diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..9d68a39241 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,18 @@ +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: +# GitHub Actions +- package-ecosystem: "github-actions" + # Workflow files stored in the + # default location of `.github/workflows` + directory: "/" + schedule: + interval: "weekly" + groups: + all-github-actions: + patterns: [ "*" ] + commit-message: + prefix: ":seedling:" + labels: + - "ok-to-test" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 2cf235d808..2bbbb33aea 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -4,7 +4,15 @@ on: types: [opened, edited, synchronize, reopened] branches: - main - - master + +permissions: + # Required: allow read access to the content for analysis. + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + pull-requests: read + # Optional: Allow write access to checks to allow the action to annotate code in the PR. + checks: write + jobs: golangci: name: lint @@ -15,9 +23,17 @@ jobs: - "" - tools/setup-envtest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + - name: Calculate go version + id: vars + run: echo "go_version=$(make go-version)" >> $GITHUB_OUTPUT + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # tag=v6.0.0 + with: + go-version: ${{ steps.vars.outputs.go_version }} - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # tag=v8.0.0 with: - version: v1.40.1 + version: v2.5.0 + args: --output.text.print-linter-name=true --output.text.colors=true --timeout 10m working-directory: ${{matrix.working-directory}} diff --git a/.github/workflows/ossf-scorecard.yaml b/.github/workflows/ossf-scorecard.yaml new file mode 100644 index 0000000000..24156f49e0 --- /dev/null +++ b/.github/workflows/ossf-scorecard.yaml @@ -0,0 +1,56 @@ +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + # Weekly on Saturdays. + - cron: '30 1 * * 6' + push: + branches: [ "main" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed if using Code scanning alerts + security-events: write + # Needed for GitHub OIDC token if publish_results is true + id-token: write + + steps: + - name: "Checkout code" + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # tag=v2.4.3 + with: + results_file: results.sarif + results_format: sarif + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + publish_results: true + + # Upload the results as artifacts. + - name: "Upload artifact" + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # tag=v4.6.2 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # required for Code scanning alerts + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@83a02f7883b12e0e4e1a146174f5e2292a01e601 # tag=v2.16.4 + with: + sarif_file: results.sarif diff --git a/.github/workflows/pr-dependabot.yaml b/.github/workflows/pr-dependabot.yaml new file mode 100644 index 0000000000..10162e9129 --- /dev/null +++ b/.github/workflows/pr-dependabot.yaml @@ -0,0 +1,38 @@ +name: PR dependabot go modules fix + +# This action runs on PRs opened by dependabot and updates modules. +on: + pull_request: + branches: + - dependabot/** + push: + branches: + - dependabot/** + workflow_dispatch: + +permissions: + contents: write # Allow to update the PR. + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + - name: Calculate go version + id: vars + run: echo "go_version=$(make go-version)" >> $GITHUB_OUTPUT + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # tag=v6.0.0 + with: + go-version: ${{ steps.vars.outputs.go_version }} + - name: Update all modules + run: make modules + - uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # tag=v9.1.4 + name: Commit changes + with: + author_name: dependabot[bot] + author_email: 49699333+dependabot[bot]@users.noreply.github.com + default_author: github_actor + message: 'Update generated code' diff --git a/.github/workflows/pr-gh-workflow-approve.yaml b/.github/workflows/pr-gh-workflow-approve.yaml new file mode 100644 index 0000000000..28be4dac71 --- /dev/null +++ b/.github/workflows/pr-gh-workflow-approve.yaml @@ -0,0 +1,42 @@ +name: PR approve GH Workflows + +on: + pull_request_target: + types: + - edited + - labeled + - reopened + - synchronize + +permissions: {} + +jobs: + approve: + name: Approve ok-to-test + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Update PR + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const result = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + event: "pull_request", + status: "action_required", + head_sha: context.payload.pull_request.head.sha, + per_page: 100 + }); + + for (var run of result.data.workflow_runs) { + await github.rest.actions.approveWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id + }); + } diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..6e3b7734e7 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,35 @@ +name: Upload binaries to release + +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + +permissions: + contents: write + +jobs: + build: + name: Upload binaries to release + runs-on: ubuntu-latest + steps: + - name: Set env + run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV + - name: Check out code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + - name: Calculate go version + id: vars + run: echo "go_version=$(make go-version)" >> $GITHUB_OUTPUT + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # tag=v6.0.0 + with: + go-version: ${{ steps.vars.outputs.go_version }} + - name: Generate release binaries + run: | + make release + - name: Release + uses: softprops/action-gh-release@62c96d0c4e8a889135c1f3a25910db8dbe0e85f7 # tag=v2.3.4 + with: + draft: false + files: tools/setup-envtest/out/* diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index caa342f44b..2168d72516 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,14 +1,18 @@ +name: PR title verifier + on: pull_request_target: - types: [opened, edited, reopened, synchronize] + types: [opened, edited, synchronize, reopened] jobs: verify: runs-on: ubuntu-latest - name: verify PR contents + steps: - - name: Verifier action - id: verifier - uses: kubernetes-sigs/kubebuilder-release-tools@v0.1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + + - name: Check if PR title is valid + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: | + ./hack/verify-pr-title.sh "${PR_TITLE}" diff --git a/.gitignore b/.gitignore index c2c72faf34..2ddc5a8b87 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,9 @@ # Tools binaries. hack/tools/bin + +# Release artifacts +tools/setup-envtest/out + +junit-report.xml +/artifacts diff --git a/.golangci.yml b/.golangci.yml index a5a5dad615..85701c88a8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,130 +1,196 @@ +version: "2" +run: + go: "1.24" + timeout: 10m + allow-parallel-runners: true linters: - disable-all: true + default: none enable: - - asciicheck - - bodyclose - - deadcode - - depguard - - dogsled - - errcheck - - exportloopref - - goconst - - gocritic - - gocyclo - - godot - - gofmt - - goimports - - goprintffuncname - - gosec - - gosimple - - govet - - ifshort - - importas - - ineffassign - - misspell - - nakedret - - nilerr - - nolintlint - - prealloc - - revive - - rowserrcheck - - staticcheck - - structcheck - - stylecheck - - typecheck - - unconvert - - unparam - - varcheck - - whitespace - -linters-settings: - ifshort: - # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. - max-decl-chars: 50 - importas: - no-unaliased: true - alias: - # Kubernetes - - pkg: k8s.io/api/core/v1 - alias: corev1 - - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 - alias: apiextensionsv1 - - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 - alias: metav1 - - pkg: k8s.io/apimachinery/pkg/api/errors - alias: apierrors - - pkg: k8s.io/apimachinery/pkg/util/errors - alias: kerrors - # Controller Runtime - - pkg: sigs.k8s.io/controller-runtime - alias: ctrl - staticcheck: - go: "1.16" - stylecheck: - go: "1.16" - -issues: - max-same-issues: 0 - max-issues-per-linter: 0 - # We are disabling default golangci exclusions because we want to help reviewers to focus on reviewing the most relevant - # changes in PRs and avoid nitpicking. - exclude-use-default: false - # List of regexps of issue texts to exclude, empty list by default. - exclude: - # The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time. - # If it is decided they will not be addressed they should be moved above this comment. - - Subprocess launch(ed with variable|ing should be audited) - - (G204|G104|G307) - - "ST1000: at least one file in a package should have a package comment" - exclude-rules: - - linters: - - gosec - text: "G108: Profiling endpoint is automatically exposed on /debug/pprof" - - linters: - - revive - text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported" - - linters: + - asasalint + - asciicheck + - bidichk + - bodyclose + - copyloopvar + - dogsled + - dupl - errcheck - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - # With Go 1.16, the new embed directive can be used with an un-named import, - # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. - # This directive allows the embed package to be imported with an underscore everywhere. - - linters: - - revive - source: _ "embed" - # Exclude some packages or code to require comments, for example test code, or fake clients. - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - source: (func|type).*Fake.* - - linters: + - errchkjson + - errorlint + - exhaustive + - forbidigo + - ginkgolinter + - goconst + - gocritic + - gocyclo + - godoclint + - goprintffuncname + - govet + - importas + - ineffassign + - iotamixing + - makezero + - misspell + - nakedret + - nilerr + - nolintlint + - prealloc - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - path: fake_\.go - # Disable unparam "always receives" which might not be really - # useful when building libraries. - - linters: + - staticcheck + - tagliatelle + - unconvert - unparam - text: always receives - # Dot imports for gomega or ginkgo are allowed - # within test files. - - path: _test\.go - text: should not use dot imports - - path: _test\.go - text: cyclomatic complexity - - path: _test\.go - text: "G107: Potential HTTP request made with variable url" - # Append should be able to assign to a different var/slice. - - linters: - - gocritic - text: "appendAssign: append result not assigned to the same slice" - - linters: - - gocritic - text: "singleCaseSwitch: should rewrite switch statement to if statement" - -run: - timeout: 10m - skip-files: - - "zz_generated.*\\.go$" - - ".*conversion.*\\.go$" - allow-parallel-runners: true + - unused + - whitespace + settings: + forbidigo: + forbid: + - pattern: context.Background + msg: Use ginkgos SpecContext or go testings t.Context instead + - pattern: context.TODO + msg: Use ginkgos SpecContext or go testings t.Context instead + govet: + disable: + - fieldalignment + - shadow + enable-all: true + importas: + alias: + - pkg: k8s.io/api/core/v1 + alias: corev1 + - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 + alias: apiextensionsv1 + - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 + alias: metav1 + - pkg: k8s.io/apimachinery/pkg/api/errors + alias: apierrors + - pkg: k8s.io/apimachinery/pkg/util/errors + alias: kerrors + - pkg: sigs.k8s.io/controller-runtime + alias: ctrl + no-unaliased: true + revive: + rules: + # The following rules are recommended https://github.com/mgechev/revive#recommended-configuration + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: if-return + - name: increment-decrement + - name: var-naming + - name: var-declaration + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: superfluous-else + - name: unreachable-code + - name: redefines-builtin-id + # + # Rules in addition to the recommended configuration above. + # + - name: bool-literal-in-expr + - name: constant-logical-expr + exclusions: + generated: strict + paths: + - zz_generated.*\.go$ + - .*conversion.*\.go$ + rules: + - linters: + - forbidigo + path-except: _test\.go + - linters: + - gosec + text: 'G108: Profiling endpoint is automatically exposed on /debug/pprof' + - linters: + - revive + text: 'exported: exported method .*\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported' + - linters: + - errcheck + text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked + - linters: + - staticcheck + text: 'SA1019: .*The component config package has been deprecated and will be removed in a future release.' + # With Go 1.16, the new embed directive can be used with an un-named import, + # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. + # This directive allows the embed package to be imported with an underscore everywhere. + - linters: + - revive + source: _ "embed" + # Exclude some packages or code to require comments, for example test code, or fake clients. + - linters: + - revive + text: exported (method|function|type|const) (.+) should have comment or be unexported + source: (func|type).*Fake.* + - linters: + - revive + path: fake_\.go + text: exported (method|function|type|const) (.+) should have comment or be unexported + # Disable unparam "always receives" which might not be really + # useful when building libraries. + - linters: + - unparam + text: always receives + # Dot imports for gomega and ginkgo are allowed + # within test files. + - path: _test\.go + text: should not use dot imports + - path: _test\.go + text: cyclomatic complexity + - path: _test\.go + text: 'G107: Potential HTTP request made with variable url' + # Append should be able to assign to a different var/slice. + - linters: + - gocritic + text: 'appendAssign: append result not assigned to the same slice' + - linters: + - gocritic + text: 'singleCaseSwitch: should rewrite switch statement to if statement' + # It considers all file access to a filename that comes from a variable problematic, + # which is naiv at best. + - linters: + - gosec + text: 'G304: Potential file inclusion via variable' + - linters: + - dupl + path: _test\.go + - linters: + - revive + path: .*/internal/.* + - linters: + - unused + # Seems to incorrectly trigger on the two implementations that are only + # used through an interface and not directly..? + # Likely same issue as https://github.com/dominikh/go-tools/issues/1616 + path: pkg/controller/priorityqueue/metrics\.go + # The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time. + # If it is decided they will not be addressed they should be moved above this comment. + - path: (.+)\.go$ + text: Subprocess launch(ed with variable|ing should be audited) + - linters: + - gosec + path: (.+)\.go$ + text: (G204|G104|G307) + - linters: + - staticcheck + path: (.+)\.go$ + text: (ST1000|QF1008) +issues: + max-issues-per-linter: 0 + max-same-issues: 0 +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: strict + paths: + - zz_generated.*\.go$ + - .*conversion.*\.go$ diff --git a/.gomodcheck.yaml b/.gomodcheck.yaml new file mode 100644 index 0000000000..3608de331d --- /dev/null +++ b/.gomodcheck.yaml @@ -0,0 +1,17 @@ +upstreamRefs: + - k8s.io/api + - k8s.io/apiextensions-apiserver + - k8s.io/apimachinery + - k8s.io/apiserver + - k8s.io/client-go + - k8s.io/component-base + # k8s.io/klog/v2 -> conflicts with k/k deps + # k8s.io/utils -> conflicts with k/k deps + +excludedModules: + # --- test dependencies: + - github.com/onsi/ginkgo/v2 + - github.com/onsi/gomega + + # --- We want a newer version with generics support for this + - github.com/google/btree diff --git a/FAQ.md b/FAQ.md index 86c7f93369..9c36c8112e 100644 --- a/FAQ.md +++ b/FAQ.md @@ -4,13 +4,13 @@ **A**: Each controller should only reconcile one object type. Other affected objects should be mapped to a single type of root object, using -the `EnqueueRequestForOwner` or `EnqueueRequestsFromMapFunc` event +the `handler.EnqueueRequestForOwner` or `handler.EnqueueRequestsFromMapFunc` event handlers, and potentially indices. Then, your Reconcile method should attempt to reconcile *all* state for that given root objects. ### Q: How do I have different logic in my reconciler for different types of events (e.g. create, update, delete)? -**A**: You should not. Reconcile functions should be idempotent, and +**A**: You should not. Reconcile functions should be idempotent, and should always reconcile state by reading all the state it needs, then writing updates. This allows your reconciler to correctly respond to generic events, adjust to skipped or coalesced events, and easily deal @@ -30,13 +30,13 @@ on your situation. take this approach: the StatefulSet controller appends a specific number to each pod that it creates, while the Deployment controller hashes the pod template spec and appends that. - + - In the few cases when you cannot take advantage of deterministic names (e.g. when using generateName), it may be useful in to track which actions you took, and assume that they need to be repeated if they don't occur after a given time (e.g. using a requeue result). This is what the ReplicaSet controller does. - + In general, write your controller with the assumption that information will eventually be correct, but may be slightly out of date. Make sure that your reconcile function enforces the entire state of the world each @@ -48,9 +48,9 @@ generally cover most circumstances. ### Q: Where's the fake client? How do I use it? **A**: The fake client -[exists](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/client/fake), +[exists](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/client/fake), but we generally recommend using -[envtest.Environment](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) +[envtest.Environment](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) to test against a real API server. In our experience, tests using fake clients gradually re-implement poorly-written impressions of a real API server, which leads to hard-to-maintain, complex test code. @@ -58,7 +58,7 @@ server, which leads to hard-to-maintain, complex test code. ### Q: How should I write tests? Any suggestions for getting started? - Use the aforementioned - [envtest.Environment](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) + [envtest.Environment](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) to spin up a real API server instead of trying to mock one out. - Structure your tests to check that the state of the world is as you @@ -77,5 +77,5 @@ mapping between Go types and group-version-kinds in Kubernetes. In general, your application should have its own Scheme containing the types from the API groups that it needs (be they Kubernetes types or your own). See the [scheme builder -docs](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/scheme) for +docs](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/scheme) for more information. diff --git a/Makefile b/Makefile index 36647c697f..b8e9cfa877 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,11 @@ SHELL:=/usr/bin/env bash .DEFAULT_GOAL:=help +# +# Go. +# +GO_VERSION ?= 1.24.0 + # Use GOPROXY environment variable if set GOPROXY := $(shell go env GOPROXY) ifeq ($(GOPROXY),) @@ -34,13 +39,22 @@ export GOPROXY # Active module mode, as we use go modules to manage dependencies export GO111MODULE=on +# Hosts running SELinux need :z added to volume mounts +SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) + +ifeq ($(SELINUX_ENABLED),1) + DOCKER_VOL_OPTS?=:z +endif + # Tools. TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(TOOLS_DIR)/bin +TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin) GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint) GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen ENVTEST_DIR := $(abspath tools/setup-envtest) +SCRATCH_ENV_DIR := $(abspath examples/scratch-env) +GO_INSTALL := ./hack/go-install.sh # The help will print out all targets with their descriptions organized bellow their categories. The categories are represented by `##@` and the target descriptions by `##`. # The awk commands is responsible to read the entire set of makefiles included in this invocation, looking for lines of the file as xyz: ## something, and then pretty-format the target and help. Then, if there's a line with ##@ something, that gets pretty-printed as a category. @@ -66,16 +80,36 @@ test-tools: ## tests the tools codebase (setup-envtest) ## Binaries ## -------------------------------------- -$(GO_APIDIFF): $(TOOLS_DIR)/go.mod # Build go-apidiff from tools folder. - cd $(TOOLS_DIR) && go build -tags=tools -o bin/go-apidiff github.com/joelanford/go-apidiff +GO_APIDIFF_VER := v0.8.2 +GO_APIDIFF_BIN := go-apidiff +GO_APIDIFF := $(abspath $(TOOLS_BIN_DIR)/$(GO_APIDIFF_BIN)-$(GO_APIDIFF_VER)) +GO_APIDIFF_PKG := github.com/joelanford/go-apidiff -$(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder. - cd $(TOOLS_DIR) && go build -tags=tools -o bin/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen +$(GO_APIDIFF): # Build go-apidiff from tools folder. + GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GO_APIDIFF_PKG) $(GO_APIDIFF_BIN) $(GO_APIDIFF_VER) -$(GOLANGCI_LINT): .github/workflows/golangci-lint.yml # Download golanci-lint using hack script into tools folder. - hack/ensure-golangci-lint.sh \ - -b $(TOOLS_BIN_DIR) \ - $(shell cat .github/workflows/golangci-lint.yml | grep version | sed 's/.*version: //') +CONTROLLER_GEN_VER := v0.17.1 +CONTROLLER_GEN_BIN := controller-gen +CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER)) +CONTROLLER_GEN_PKG := sigs.k8s.io/controller-tools/cmd/controller-gen + +$(CONTROLLER_GEN): # Build controller-gen from tools folder. + GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONTROLLER_GEN_PKG) $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER) + +GOLANGCI_LINT_BIN := golangci-lint +GOLANGCI_LINT_VER := $(shell cat .github/workflows/golangci-lint.yml | grep [[:space:]]version: | sed 's/.*version: //') +GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER)) +GOLANGCI_LINT_PKG := github.com/golangci/golangci-lint/v2/cmd/golangci-lint + +$(GOLANGCI_LINT): # Build golangci-lint from tools folder. + GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOLANGCI_LINT_PKG) $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + +GO_MOD_CHECK_DIR := $(abspath ./hack/tools/cmd/gomodcheck) +GO_MOD_CHECK := $(abspath $(TOOLS_BIN_DIR)/gomodcheck) +GO_MOD_CHECK_IGNORE := $(abspath .gomodcheck.yaml) +.PHONY: $(GO_MOD_CHECK) +$(GO_MOD_CHECK): # Build gomodcheck + go build -C $(GO_MOD_CHECK_DIR) -o $(GO_MOD_CHECK) ## -------------------------------------- ## Linting @@ -99,10 +133,49 @@ modules: ## Runs go mod to ensure modules are up to date. go mod tidy cd $(TOOLS_DIR); go mod tidy cd $(ENVTEST_DIR); go mod tidy + cd $(SCRATCH_ENV_DIR); go mod tidy -.PHONY: generate -generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config file - $(CONTROLLER_GEN) object paths="./pkg/config/v1alpha1/...;./examples/configfile/custom/v1alpha1/..." +## -------------------------------------- +## Release +## -------------------------------------- + +RELEASE_DIR := tools/setup-envtest/out + +.PHONY: $(RELEASE_DIR) +$(RELEASE_DIR): + mkdir -p $(RELEASE_DIR)/ + +.PHONY: release +release: clean-release $(RELEASE_DIR) ## Build release. + @if ! [ -z "$$(git status --porcelain)" ]; then echo "Your local git repository contains uncommitted changes, use git clean before proceeding."; exit 1; fi + + # Build binaries first. + $(MAKE) release-binaries + +.PHONY: release-binaries +release-binaries: ## Build release binaries. + RELEASE_BINARY=setup-envtest-linux-amd64 GOOS=linux GOARCH=amd64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-linux-arm64 GOOS=linux GOARCH=arm64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-linux-ppc64le GOOS=linux GOARCH=ppc64le $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-linux-s390x GOOS=linux GOARCH=s390x $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-darwin-amd64 GOOS=darwin GOARCH=amd64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-darwin-arm64 GOOS=darwin GOARCH=arm64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-windows-amd64.exe GOOS=windows GOARCH=amd64 $(MAKE) release-binary + +.PHONY: release-binary +release-binary: $(RELEASE_DIR) + docker run \ + --rm \ + -e CGO_ENABLED=0 \ + -e GOOS=$(GOOS) \ + -e GOARCH=$(GOARCH) \ + -e GOCACHE=/tmp/ \ + --user $$(id -u):$$(id -g) \ + -v "$$(pwd):/workspace$(DOCKER_VOL_OPTS)" \ + -w /workspace/tools/setup-envtest \ + golang:$(GO_VERSION) \ + go build -a -trimpath -ldflags "-X 'sigs.k8s.io/controller-runtime/tools/setup-envtest/version.version=$(RELEASE_TAG)' -extldflags '-static'" \ + -o ./out/$(RELEASE_BINARY) ./ ## -------------------------------------- ## Cleanup / Verification @@ -110,14 +183,36 @@ generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config .PHONY: clean clean: ## Cleanup. + $(GOLANGCI_LINT) cache clean $(MAKE) clean-bin .PHONY: clean-bin clean-bin: ## Remove all generated binaries. rm -rf hack/tools/bin +.PHONY: clean-release +clean-release: ## Remove the release folder + rm -rf $(RELEASE_DIR) + .PHONY: verify-modules -verify-modules: modules - @if !(git diff --quiet HEAD -- go.sum go.mod); then \ +verify-modules: modules $(GO_MOD_CHECK) ## Verify go modules are up to date + @if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(ENVTEST_DIR)/go.mod $(ENVTEST_DIR)/go.sum $(SCRATCH_ENV_DIR)/go.sum); then \ + git diff; \ echo "go module files are out of date, please run 'make modules'"; exit 1; \ fi + $(GO_MOD_CHECK) $(GO_MOD_CHECK_IGNORE) + +APIDIFF_OLD_COMMIT ?= $(shell git rev-parse origin/main) + +.PHONY: apidiff +verify-apidiff: $(GO_APIDIFF) ## Check for API differences + $(GO_APIDIFF) $(APIDIFF_OLD_COMMIT) --print-compatible + +## -------------------------------------- +## Helpers +## -------------------------------------- + +##@ helpers: + +go-version: ## Print the go version we use to compile our binaries and images + @echo $(GO_VERSION) diff --git a/OWNERS b/OWNERS index 4b1fa044bf..9f2d296e4c 100644 --- a/OWNERS +++ b/OWNERS @@ -6,5 +6,6 @@ approvers: - controller-runtime-approvers reviewers: - controller-runtime-admins - - controller-runtime-reviewers + - controller-runtime-maintainers - controller-runtime-approvers + - controller-runtime-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 290dcc192e..47bf6eedf3 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -4,37 +4,36 @@ aliases: # active folks who can be contacted to perform admin-related # tasks on the repo, or otherwise approve any PRS. controller-runtime-admins: - - droot - - mengqiy - - pwittrock + - alvaroaleman + - joelanford + - sbueringer + - vincepri # non-admin folks who have write-access and can approve any PRs in the repo controller-runtime-maintainers: - - vincepri + - alvaroaleman - joelanford + - sbueringer + - vincepri # non-admin folks who can approve any PRs in the repo controller-runtime-approvers: - - gerred - - shawn-hurley - - alvaroaleman + - fillzpp # folks who can review and LGTM any PRs in the repo (doesn't # include approvers & admins -- those count too via the OWNERS # file) controller-runtime-reviewers: - - alenkacz - - vincepri - - alexeldeib - varshaprasad96 - - # folks to can approve things in the directly-ported - # testing_frameworks portions of the codebase - testing-integration-approvers: - - apelisse - - hoegaarden + - inteon + - JoelSpeed + - troy0820 # folks who may have context on ancient history, # but are no longer directly involved controller-runtime-emeritus-maintainers: - directxman12 + controller-runtime-emeritus-admins: + - droot + - mengqiy + - pwittrock diff --git a/README.md b/README.md index d857d3e76a..54bacad42e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Go Report Card](https://goreportcard.com/badge/sigs.k8s.io/controller-runtime)](https://goreportcard.com/report/sigs.k8s.io/controller-runtime) -[![godoc](https://godoc.org/sigs.k8s.io/controller-runtime?status.svg)](https://godoc.org/sigs.k8s.io/controller-runtime) +[![godoc](https://pkg.go.dev/badge/sigs.k8s.io/controller-runtime)](https://pkg.go.dev/sigs.k8s.io/controller-runtime) # Kubernetes controller-runtime Project @@ -16,8 +16,8 @@ Documentation: - [Basic controller using builder](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/builder#example-Builder) - [Creating a manager](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#example-New) - [Creating a controller](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#example-New) -- [Examples](https://github.com/kubernetes-sigs/controller-runtime/blob/master/examples) -- [Designs](https://github.com/kubernetes-sigs/controller-runtime/blob/master/designs) +- [Examples](https://github.com/kubernetes-sigs/controller-runtime/blob/main/examples) +- [Designs](https://github.com/kubernetes-sigs/controller-runtime/blob/main/designs) # Versioning, Maintenance, and Compatibility @@ -25,9 +25,9 @@ The full documentation can be found at [VERSIONING.md](VERSIONING.md), but TL;DR Users: -- We follow [Semantic Versioning (semver)](https://semver.org) -- Use releases with your dependency management to ensure that you get compatible code -- The master branch contains all the latest code, some of which may break compatibility (so "normal" `go get` is not recommended) +- We stick to a zero major version +- We publish a minor version for each Kubernetes minor release and allow breaking changes between minor versions +- We publish patch versions as needed and we don't allow breaking changes in them Contributors: @@ -40,6 +40,27 @@ Contributors: * [Documentation Changes](/.github/PULL_REQUEST_TEMPLATE/docs.md) * [Test/Build/Other Changes](/.github/PULL_REQUEST_TEMPLATE/other.md) +## Compatibility + +Every minor version of controller-runtime has been tested with a specific minor version of client-go. A controller-runtime minor version *may* be compatible with +other client-go minor versions, but this is by chance and neither supported nor tested. In general, we create one minor version of controller-runtime +for each minor version of client-go and other k8s.io/* dependencies. + +The minimum Go version of controller-runtime is the highest minimum Go version of our Go dependencies. Usually, this will +be identical to the minimum Go version of the corresponding k8s.io/* dependencies. + +Compatible k8s.io/*, client-go and minimum Go versions can be looked up in our [go.mod](go.mod) file. + +| | k8s.io/*, client-go | minimum Go version | +|----------|:-------------------:|:------------------:| +| CR v0.21 | v0.33 | 1.24 | +| CR v0.20 | v0.32 | 1.23 | +| CR v0.19 | v0.31 | 1.22 | +| CR v0.18 | v0.30 | 1.22 | +| CR v0.17 | v0.29 | 1.21 | +| CR v0.16 | v0.28 | 1.20 | +| CR v0.15 | v0.27 | 1.20 | + ## FAQ See [FAQ.md](FAQ.md) @@ -48,15 +69,13 @@ See [FAQ.md](FAQ.md) Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). -controller-runtime is a subproject of the [kubebuilder](https://github.com/kubernetes-sigs/kubebuilder) project -in sig apimachinery. - You can reach the maintainers of this project at: -- Slack channel: [#kubebuilder](http://slack.k8s.io/#kubebuilder) +- Slack channel: [#controller-runtime](https://kubernetes.slack.com/archives/C02MRBMN00Z) - Google Group: [kubebuilder@googlegroups.com](https://groups.google.com/forum/#!forum/kubebuilder) ## Contributing + Contributions are greatly appreciated. The maintainers actively manage the issues list, and try to highlight issues suitable for newcomers. The project follows the typical GitHub pull request model. See [CONTRIBUTING.md](CONTRIBUTING.md) for more details. Before starting any work, please either comment on an existing issue, or file a new one. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000000..2a857b976e --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,51 @@ +# Release Process + +The Kubernetes controller-runtime Project is released on an as-needed basis. The process is as follows: + +**Note:** Releases are done from the `release-MAJOR.MINOR` branches. For PATCH releases is not required +to create a new branch you will just need to ensure that all big fixes are cherry-picked into the respective +`release-MAJOR.MINOR` branch. To know more about versioning check https://semver.org/. + +## How to do a release + +### Create the new branch and the release tag + +1. Create a new branch `git checkout -b release-` from main +2. Push the new branch to the remote repository + +### Now, let's generate the changelog + +1. Create the changelog from the new branch `release-` (`git checkout release-`). +You will need to use the [kubebuilder-release-tools][kubebuilder-release-tools] to generate the notes. See [here][release-notes-generation] + +> **Note** +> - You will need to have checkout locally from the remote repository the previous branch +> - Also, ensure that you fetch all tags from the remote `git fetch --all --tags` + +### Draft a new release from GitHub + +1. Create a new tag with the correct version from the new `release-` branch +2. Add the changelog on it and publish. Now, the code source is released ! + +### Add a new Prow test the for the new branch release + +1. Create a new prow test under [github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime](https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime) +for the new `release-` branch. (i.e. for the `0.11.0` release see the PR: https://github.com/kubernetes/test-infra/pull/25205) +2. Ping the infra PR in the controller-runtime slack channel for reviews. + +### Announce the new release: + +1. Publish on the Slack channel the new release, i.e: + +```` +:announce: Controller-Runtime v0.12.0 has been released! +This release includes a Kubernetes dependency bump to v1.24. +For more info, see the release page: https://github.com/kubernetes-sigs/controller-runtime/releases. + :tada: Thanks to all our contributors! +```` + +2. An announcement email is sent to `kubebuilder@googlegroups.com` with the subject `[ANNOUNCE] Controller-Runtime $VERSION is released` + +[kubebuilder-release-tools]: https://github.com/kubernetes-sigs/kubebuilder-release-tools +[release-notes-generation]: https://github.com/kubernetes-sigs/kubebuilder-release-tools/blob/master/README.md#release-notes-generation +[release-process]: https://github.com/kubernetes-sigs/kubebuilder/blob/master/VERSIONING.md#releasing diff --git a/SECURITY_CONTACTS b/SECURITY_CONTACTS index 32e6a3b904..9c5241c6b4 100644 --- a/SECURITY_CONTACTS +++ b/SECURITY_CONTACTS @@ -10,5 +10,6 @@ # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE # INSTRUCTIONS AT https://kubernetes.io/security/ -pwittrock -droot +alvaroaleman +sbueringer +vincepri diff --git a/TMP-LOGGING.md b/TMP-LOGGING.md index b3cfc66517..97e091fd48 100644 --- a/TMP-LOGGING.md +++ b/TMP-LOGGING.md @@ -21,7 +21,7 @@ log.Printf("starting reconciliation for pod %s/%s", podNamespace, podName) In controller-runtime, we'd instead write: ```go -logger.Info("starting reconciliation", "pod", req.NamespacedNamed) +logger.Info("starting reconciliation", "pod", req.NamespacedName) ``` or even write @@ -51,7 +51,7 @@ You can configure the logging implementation using `"sigs.k8s.io/controller-runtime/pkg/log".SetLogger`. That package also contains the convenience functions for setting up Zap. -You can get a handle to the the "root" logger using +You can get a handle to the "root" logger using `"sigs.k8s.io/controller-runtime/pkg/log".Log`, and can then call `WithName` to create individual named loggers. You can call `WithName` repeatedly to chain names together: diff --git a/VERSIONING.md b/VERSIONING.md index 18779000ec..7ad6b142cc 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -7,9 +7,19 @@ For the purposes of the aforementioned guidelines, controller-runtime counts as a "library project", but otherwise follows the guidelines exactly. +We stick to a major version of zero and create a minor version for +each Kubernetes minor version and we allow breaking changes in our +minor versions. We create patch releases as needed and don't allow +breaking changes in them. + +Publishing a non-zero major version is pointless for us, as the k8s.io/* +libraries we heavily depend on do breaking changes but use the same +versioning scheme as described above. Consequently, a project can only +ever depend on one controller-runtime version. + [guidelines]: https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md -## Compatiblity and Release Support +## Compatibility and Release Support For release branches, we generally tend to support backporting one (1) major release (`release-{X-1}` or `release-0.{Y-1}`), but may go back @@ -19,12 +29,12 @@ further if the need arises and is very pressing (e.g. security updates). Note the [guidelines on dependency versions][dep-versions]. Particularly: -- We **DO** guarantee Kubernetes REST API compability -- if a given +- We **DO** guarantee Kubernetes REST API compatibility -- if a given version of controller-runtime stops working with what should be a supported version of Kubernetes, this is almost certainly a bug. -- We **DO NOT** guarantee any particular compability matrix between +- We **DO NOT** guarantee any particular compatibility matrix between kubernetes library dependencies (client-go, apimachinery, etc); Such - compability is infeasible due to the way those libraries are versioned. + compatibility is infeasible due to the way those libraries are versioned. [dep-versions]: https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md#kubernetes-version-compatibility diff --git a/alias.go b/alias.go index 29f964dcbe..01ba012dcc 100644 --- a/alias.go +++ b/alias.go @@ -21,7 +21,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client/config" - cfg "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -70,10 +69,17 @@ type TypeMeta = metav1.TypeMeta type ObjectMeta = metav1.ObjectMeta var ( + // RegisterFlags registers flag variables to the given FlagSet if not already registered. + // It uses the default command line FlagSet, if none is provided. Currently, it only registers the kubeconfig flag. + RegisterFlags = config.RegisterFlags + // GetConfigOrDie creates a *rest.Config for talking to a Kubernetes apiserver. // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // + // The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and + // fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. + // // Will log an error and exit if there is an error creating the rest.Config. GetConfigOrDie = config.GetConfigOrDie @@ -81,6 +87,9 @@ var ( // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // + // The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and + // fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. + // // Config precedence // // * --kubeconfig flag pointing at a file @@ -92,11 +101,6 @@ var ( // * $HOME/.kube/config if exists. GetConfig = config.GetConfig - // ConfigFile returns the cfg.File function for deferred config file loading, - // this is passed into Options{}.From() to populate the Options fields for - // the manager. - ConfigFile = cfg.File - // NewControllerManagedBy returns a new controller builder that will be started by the provided Manager. NewControllerManagedBy = builder.ControllerManagedBy @@ -104,6 +108,9 @@ var ( NewWebhookManagedBy = builder.WebhookManagedBy // NewManager returns a new Manager for creating Controllers. + // Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf" + // will be used for all built-in resources of Kubernetes, and "application/json" is for other types + // including all CRD resources. NewManager = manager.New // CreateOrUpdate creates or updates the given object obj in the Kubernetes @@ -121,8 +128,8 @@ var ( // there is another OwnerReference with Controller flag set. SetControllerReference = controllerutil.SetControllerReference - // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned - // which is closed on one of these signals. If a second signal is caught, the program + // SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned + // which is canceled on one of these signals. If a second signal is caught, the program // is terminated with exit code 1. SetupSignalHandler = signals.SetupSignalHandler @@ -135,7 +142,7 @@ var ( // The logger, when used with controllers, can be expected to contain basic information about the object // that's being reconciled like: // - `reconciler group` and `reconciler kind` coming from the For(...) object passed in when building a controller. - // - `name` and `namespace` injected from the reconciliation request. + // - `name` and `namespace` from the reconciliation request. // // This is meant to be used with the context supplied in a struct that satisfies the Reconciler interface. LoggerFrom = log.FromContext diff --git a/designs/README.md b/designs/README.md index 3ed7de5ad3..bf8b5000a9 100644 --- a/designs/README.md +++ b/designs/README.md @@ -1,12 +1,12 @@ Designs ======= -These are design documents for changes to Controller Runtime They exist -to help document the design processes that go into writing Controller -Runtime, but may not be up-to-date (more below). +These are the design documents for changes to Controller Runtime. They +exist to help document the design processes that go into writing +Controller Runtime, but may not be up-to-date (more below). Not all changes to Controller Runtime need a design document -- only major -ones. use your best judgement. +ones. Use your best judgement. When submitting a design document, we encourage having written a proof-of-concept, and it's perfectly acceptable to submit the @@ -17,7 +17,7 @@ proof-of-concept process can help iron out wrinkles and can help with the ## Out-of-Date Designs **Controller Runtime documentation -[GoDoc](https://godoc.org/sigs.k8s.io/controller-runtime) should be +[GoDoc](https://pkg.go.dev/sigs.k8s.io/controller-runtime) should be considered the canonical, update-to-date reference and architectural documentation** for Controller Runtime. @@ -29,8 +29,8 @@ why things changed. For example: # Out of Date -This change is out of date. It turns out curly braces a frustrating to +This change is out of date. It turns out curly braces are frustrating to type, so we had to abandon functions entirely, and have users specify custom functionality using strings of Common LISP instead. See #000 for more information. -``` \ No newline at end of file +``` diff --git a/designs/cache_options.md b/designs/cache_options.md new file mode 100644 index 0000000000..bdd29c0481 --- /dev/null +++ b/designs/cache_options.md @@ -0,0 +1,226 @@ +Cache Options +=================== + +This document describes how we imagine the cache options to look in +the future. + +## Goals + +* Align everyone on what settings on the cache we want to support and + their configuration surface +* Ensure that we support both complicated cache setups and provide an + intuitive configuration UX + +## Non-Goals + +* Describe the design and implementation of the cache itself. + The assumption is that the most granular level we will end up with is + "per-object multiple namespaces with distinct selectors" and that this + can be implemented using a "meta cache" that delegates per object and by + extending the current multi-namespace cache +* Outline any kind of timeline for when these settings will be implemented. + Implementation will happen gradually over time whenever someone steps up + to do the actual work + +## Proposal + + +``` +const ( + AllNamespaces = corev1.NamespaceAll +) + +type Config struct { + // LabelSelector specifies a label selector. A nil value allows to + // default this. + LabelSelector labels.Selector + + // FieldSelector specifics a field selector. A nil value allows to + // default this. + FieldSelector fields.Selector + + // Transform specifies a transform func. A nil value allows to default + // this. + Transform toolscache.TransformFunc + + // UnsafeDisableDeepCopy specifies if List and Get requests against the + // cache should not DeepCopy. A nil value allows to default this. + UnsafeDisableDeepCopy *bool +} + + +type ByObject struct { + // Namespaces maps a namespace name to cache setting. If set, only the + // namespaces in this map will be cached. + // + // Settings in the map value that are unset because either the value as a + // whole is nil or because the specific setting is nil will be defaulted. + // Use an empty value for the specific setting to prevent that. + // + // It is possible to have specific Config for just some namespaces + // but cache all namespaces by using the AllNamespaces const as the map key. + // This wil then include all namespaces that do not have a more specific + // setting. + // + // A nil map allows to default this to the cache's DefaultNamespaces setting. + // An empty map prevents this. + // + // This must be unset for cluster-scoped objects. + Namespaces map[string]Config + + // Config will be used for cluster-scoped objects and to default + // Config in the Namespaces field. + // + // It gets defaulted from the cache'sDefaultLabelSelector, DefaultFieldSelector, + // DefaultUnsafeDisableDeepCopy and DefaultTransform. + Config *Config +} + +type Options struct { + // ByObject specifies per-object cache settings. If unset for a given + // object, this will fall through to Default* settings. + ByObject map[client.Object]ByObject + + // DefaultNamespaces maps namespace names to cache settings. If set, it + // will be used for all objects that have a nil Namespaces setting. + // + // It is possible to have a specific Config for just some namespaces + // but cache all namespaces by using the `AllNamespaces` const as the map + // key. This wil then include all namespaces that do not have a more + // specific setting. + // + // The options in the Config that are nil will be defaulted from + // the respective Default* settings. + DefaultNamespaces map[string]Config + + // DefaultLabelSelector is the label selector that will be used as + // the default field label selector for everything that doesn't + // have one configured. + DefaultLabelSelector labels.Selector + + // DefaultFieldSelector is the field selector that will be used as + // the default field selector for everything that doesn't have + // one configured. + DefaultFieldSelector fields.Selector + + // DefaultUnsafeDisableDeepCopy is the default for UnsafeDisableDeepCopy + // for everything that doesn't specify this. + DefaultUnsafeDisableDeepCopy *bool + + // DefaultTransform will be used as transform for all object types + // unless they have a more specific transform set in ByObject. + DefaultTransform toolscache.TransformFunc + + // HTTPClient is the http client to use for the REST client + HTTPClient *http.Client + + // Scheme is the scheme to use for mapping objects to GroupVersionKinds + Scheme *runtime.Scheme + + // Mapper is the RESTMapper to use for mapping GroupVersionKinds to Resources + Mapper meta.RESTMapper + + // SyncPeriod determines the minimum frequency at which watched resources are + // reconciled. A lower period will correct entropy more quickly, but reduce + // responsiveness to change if there are many watched resources. Change this + // value only if you know what you are doing. Defaults to 10 hours if unset. + // there will a 10 percent jitter between the SyncPeriod of all controllers + // so that all controllers will not send list requests simultaneously. + // + // This applies to all controllers. + // + // A period sync happens for two reasons: + // 1. To insure against a bug in the controller that causes an object to not + // be requeued, when it otherwise should be requeued. + // 2. To insure against an unknown bug in controller-runtime, or its dependencies, + // that causes an object to not be requeued, when it otherwise should be + // requeued, or to be removed from the queue, when it otherwise should not + // be removed. + // + // If you want + // 1. to insure against missed watch events, or + // 2. to poll services that cannot be watched, + // then we recommend that, instead of changing the default period, the + // controller requeue, with a constant duration `t`, whenever the controller + // is "done" with an object, and would otherwise not requeue it, i.e., we + // recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`, + // instead of `reconcile.Result{}`. + SyncPeriod *time.Duration + +} +``` + + +## Example usages + +### Cache ConfigMaps in the `public` and `kube-system` namespaces and Secrets in the `operator` Namespace + + +``` +cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.ConfigMap{}: { + Namespaces: map[string]cache.Config{ + "public": {}, + "kube-system": {}, + }, + }, + &corev1.Secret{}: {Namespaces: map[string]Config{ + "operator": {}, + }}, + }, +} +``` + +### Cache ConfigMaps in all namespaces without selector, but have a selector for the `operator` Namespace + +``` +cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.ConfigMap{}: { + Namespaces: map[string]cache.Config{ + cache.AllNamespaces: nil, // No selector for all namespaces... + "operator": {LabelSelector: labelSelector}, // except for the operator namespace + }, + }, + }, +} +``` + + +### Only cache the `operator` namespace for namespaced objects and all namespaces for Deployments + +``` +cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &appsv1.Deployment: {Namespaces: map[string]cache.Config{ + cache.AllNamespaces: {}}, + }}, + }, + DefaultNamespaces: map[string]cache.Config{ + "operator": {}}, + }, +} +``` + +### Use a LabelSelector for everything except Nodes + +``` +cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Node: {LabelSelector: labels.Everything()}, + }, + DefaultLabelSelector: myLabelSelector, +} +``` + +### Only cache namespaced objects in the `foo` and `bar` namespace + +``` +cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + "foo": {}, + "bar": {}, + } +} +``` diff --git a/designs/component-config.md b/designs/component-config.md index 00d6d59391..8aebec4f96 100644 --- a/designs/component-config.md +++ b/designs/component-config.md @@ -37,7 +37,7 @@ Currently controllers that use `controller-runtime` need to configure the `ctrl. ## Motivation -This change is important because: +This change is important because: - it will help make it easier for controllers to be configured by other machine processes - it will reduce the required flags required to start a controller - allow for configuration types which aren't natively supported by flags @@ -65,7 +65,7 @@ This change is important because: ## Proposal -The `ctrl.Manager` _SHOULD_ support loading configurations from `ComponentConfig` like objects. +The `ctrl.Manager` _SHOULD_ support loading configurations from `ComponentConfig` like objects. An interface for that object with getters for the specific configuration parameters is created to bridge existing patterns. Without breaking the current `ctrl.NewManager` which uses an exported `ctrl.Options{}` the `manager.go` can expose a new func, `NewFromComponentConfig()` this would be able to loop through the getters to populate an internal `ctrl.Options{}` and pass that into `New()`. @@ -101,7 +101,7 @@ type ManagerConfiguration interface { func NewFromComponentConfig(config *rest.Config, scheme *runtime.Scheme, filename string, managerconfig ManagerConfiguration) (Manager, error) { codecs := serializer.NewCodecFactory(scheme) if err := decodeComponentConfigFileInto(codecs, filename, managerconfig); err != nil { - + } options := Options{} @@ -139,7 +139,7 @@ import ( // ControllerManagerConfiguration defines the embedded RuntimeConfiguration for controller-runtime clients. type ControllerManagerConfiguration struct { - Namespace string `json:"namespace,omitempty"` + Namespace string `json:"namespace,omitempty"` SyncPeriod *time.Duration `json:"syncPeriod,omitempty"` @@ -168,7 +168,7 @@ type ControllerManagerConfigurationHealth struct { #### Default ComponentConfig Type -To enable `controller-runtime` to have a default `ComponentConfig` struct which can be used instead of requiring each controller or extension to build it's own `ComponentConfig` type, we can create a `DefaultControllerConfiguration` type which can exist in `pkg/api/config/v1alpha1/types.go`. This will allow the controller authors to use this before needing to implement their own type with additional configs. +To enable `controller-runtime` to have a default `ComponentConfig` struct which can be used instead of requiring each controller or extension to build its own `ComponentConfig` type, we can create a `DefaultControllerConfiguration` type which can exist in `pkg/api/config/v1alpha1/types.go`. This will allow the controller authors to use this before needing to implement their own type with additional configs. ```golang // pkg/api/config/v1alpha1/types.go @@ -212,12 +212,12 @@ if err != nil { } ``` -The above example uses `configname` which is the name of the file to load the configuration from and uses `scheme` to get the specific serializer, eg `serializer.NewCodecFactory(scheme)`. This will allow the configuration to be unmarshalled into the `runtime.Object` type and passed into the +The above example uses `configname` which is the name of the file to load the configuration from and uses `scheme` to get the specific serializer, eg `serializer.NewCodecFactory(scheme)`. This will allow the configuration to be unmarshalled into the `runtime.Object` type and passed into the `ctrl.NewManagerFromComponentConfig()` as a `ManagerConfiguration` interface. #### Using Flags w/ ComponentConfig -Since this design still requires setting up the initial `ComponentConfig` type and passing in a pointer to `ctrl.NewFromComponentConfig()` if you want to allow for the use of flags, your controller can use any of the different flagging interfaces. eg [`flag`](https://golang.org/pkg/flag/), [`pflag`](https://godoc.org/github.com/spf13/pflag), [`flagnum`](https://godoc.org/github.com/luci/luci-go/common/flag/flagenum) and set values on the `ComponentConfig` type prior to passing the pointer into the `ctrl.NewFromComponentConfig()`, example below. +Since this design still requires setting up the initial `ComponentConfig` type and passing in a pointer to `ctrl.NewFromComponentConfig()` if you want to allow for the use of flags, your controller can use any of the different flagging interfaces. eg [`flag`](https://golang.org/pkg/flag/), [`pflag`](https://pkg.go.dev/github.com/spf13/pflag), [`flagnum`](https://pkg.go.dev/github.com/luci/luci-go/common/flag/flagenum) and set values on the `ComponentConfig` type prior to passing the pointer into the `ctrl.NewFromComponentConfig()`, example below. ```golang leaderElect := true @@ -247,7 +247,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1alpha1 "sigs.k8s.io/controller-runtime/pkg/apis/config/v1alpha1" -) +) type ControllerNameConfigurationSpec struct { configv1alpha1.ControllerManagerConfiguration `json:",inline"` diff --git a/designs/priorityqueue.md b/designs/priorityqueue.md new file mode 100644 index 0000000000..ef1f7588a6 --- /dev/null +++ b/designs/priorityqueue.md @@ -0,0 +1,110 @@ +Priority Queue +=================== + +This document describes the motivation behind implementing a priority queue +in controller-runtime and its design details. + +## Motivation + +1. Controllers reconcile all objects during startup to account for changes in + the reconciliation logic. Some controllers also periodically re-reconcile + everything to account for out of band changes they do not get notified for, + this is for example common for controllers managing cloud resources. In both + these cases, the reconciliation of new or changed objects gets delayed, + resulting in poor user experience. [Example][0] +2. There may be application-specific reason why some events are more important + than others, [Example][1] + +## Proposed changes + +Implement a priority queue in controller-runtime that exposes the following +interface: + +```go +type PriorityQueue[T comparable] interface { + // AddWithOpts adds one or more items to the workqueue. Items + // in the workqueue are de-duplicated, so there will only ever + // be one entry for a given key. + // Adding an item that is already there may update its wait + // period to the lowest of existing and new wait period or + // its priority to the highest of existing and new priority. + AddWithOpts(o AddOpts, items ...T) + + // GetWithPriority returns an item and its priority. It allows + // a controller to re-use the priority if it enqueues an item + // again. + GetWithPriority() (item T, priority int, shutdown bool) + + // workqueue.TypedRateLimitingInterface is kept for backwards + // compatibility. + workqueue.TypedRateLimitingInterface[T] +} + +type AddOpts struct { + // After is a duration after which the object will be available for + // reconciliation. If the object is already in the workqueue, the + // lowest of existing and new After period will be used. + After time.Duration + + // Ratelimited specifies if the ratelimiter should be used to + // determine a wait period. If the object is already in the + // workqueue, the lowest of existing and new wait period will be + // used. + RateLimited bool + + // Priority specifies the priority of the object. Objects with higher + // priority are returned before objects with lower priority. If the + // object is already in the workqueue, the priority will be updated + // to the highest of existing and new priority. + // + // The default value is 0. + Priority int +} +``` + +In order to fix the issue described in point one of the motivation section, +we have to be able to differentiate events stemming from the initial list +during startup and from resyncs from other events. For events from the initial +list, the informer emits a `Create` event whereas for `Resync` it emits an `Update` +event. The suggestion is to use a heuristic for `Create` events, if the object +in there is older than one minute, it is assumed to be from the initial `List`. +For the `Resync`, we simply check if the `ResourceVersion` is unchanged. +In both these cases, we will lower the priority to `LowPriority`/`-100`. +This gives some room for use-cases where people want to use a priority that +is lower than default (`0`) but higher than what we use in the wrapper. + +```go +// WithLowPriorityWhenUnchanged wraps an existing handler and will +// reduce the priority of events stemming from the initial listwatch +// or cache resyncs to LowPriority. +func WithLowPriorityWhenUnchanged[object client.Object, request comparable](u TypedEventHandler[object, request]) TypedEventHandler[object, request]{ +} +``` + +```go +// LowPriority is the priority set by WithLowPriorityWhenUnchanged +const LowPriority = -100 +``` + +The issue described in point two of the motivation section ("application-specific +reasons to prioritize some events") will always require implementation of a custom +handler or eventsource in order to inject the appropriate priority. + +## Implementation stages + +In order to safely roll this out to all controller-runtime users, it is suggested to +divide the implementation into two stages: Initially, we will add the priority queue +but mark it as experimental and all usage of it requires explicit opt-in by setting +a boolean on the manager or configuring `NewQueue` in a controllers opts. There will +be no breaking changes required for this, but sources or handlers that want to make +use of the new queue will have to use type assertions. + +After we've gained some confidence that the implementation is useful and correct, we +will make it the default. Doing so entails breaking the `source.Source` and the +`handler.Handler` interfaces as well as the `controller.Options` struct to refer to +the new workqueue interface. We will wait at least one minor release after introducing +the `PriorityQueue` before doing this. + + +* [0]: https://youtu.be/AYNaaXlV8LQ?si=i2Pfo7Ske6rTrPLS +* [1]: https://github.com/cilium/cilium/blob/a17d6945b29c177209af3d985bd82cce49eed4a1/operator/pkg/ciliumendpointslice/controller.go#L73 diff --git a/designs/use-selectors-at-cache.md b/designs/use-selectors-at-cache.md index c9175e9d38..1d7ec6ecfb 100644 --- a/designs/use-selectors-at-cache.md +++ b/designs/use-selectors-at-cache.md @@ -1,5 +1,5 @@ -Filter cache ListWatch using selectors -=================== +# Filter cache ListWatch using selectors + ## Motivation Controller-Runtime controllers use a cache to subscribe to events from @@ -32,6 +32,7 @@ This proposal is related to the following issue [2] Add a new selector code at `pkg/cache/internal/selector.go` with common structs and helpers + ```golang package internal @@ -53,14 +54,12 @@ type Selector struct { func (s Selector) ApplyToList(listOpts *metav1.ListOptions) { ... } - +``` Add a type alias to `pkg/cache/cache.go` to internal ```golang - type SelectorsByObject internal.SelectorsByObject - ``` Extend `cache.Options` as follows: @@ -79,7 +78,6 @@ Add new builder function that will return a cache constructor using the passed cache.Options, users can set SelectorsByObject there to filter out cache, it will convert SelectorByObject to SelectorsByGVK - ```golang func BuilderWithOptions(options cache.Options) NewCacheFunc { ... @@ -119,7 +117,6 @@ implications of using a different cache than the default one ) ``` - [1] https://github.com/nmstate/kubernetes-nmstate/pull/687 [2] https://github.com/kubernetes-sigs/controller-runtime/issues/244 [3] https://github.com/kubernetes-sigs/controller-runtime/pull/1404 diff --git a/doc.go b/doc.go index 429eb129f9..75d1d908c5 100644 --- a/doc.go +++ b/doc.go @@ -23,13 +23,14 @@ limitations under the License. // and uncommon cases should be possible. In general, controller-runtime tries // to guide users towards Kubernetes controller best-practices. // -// Getting Started +// # Getting Started // // The main entrypoint for controller-runtime is this root package, which // contains all of the common types needed to get started building controllers: -// import ( -// ctrl "sigs.k8s.io/controller-runtime" -// ) +// +// import ( +// ctrl "sigs.k8s.io/controller-runtime" +// ) // // The examples in this package walk through a basic controller setup. The // kubebuilder book (https://book.kubebuilder.io) has some more in-depth @@ -38,25 +39,25 @@ limitations under the License. // controller-runtime favors structs with sane defaults over constructors, so // it's fairly common to see structs being used directly in controller-runtime. // -// Organization +// # Organization // // A brief-ish walkthrough of the layout of this library can be found below. Each // package contains more information about how to use it. // // Frequently asked questions about using controller-runtime and designing // controllers can be found at -// https://github.com/kubernetes-sigs/controller-runtime/blob/master/FAQ.md. +// https://github.com/kubernetes-sigs/controller-runtime/blob/main/FAQ.md. // -// Managers +// # Managers // // Every controller and webhook is ultimately run by a Manager (pkg/manager). A // manager is responsible for running controllers and webhooks, and setting up -// common dependencies (pkg/runtime/inject), like shared caches and clients, as +// common dependencies, like shared caches and clients, as // well as managing leader election (pkg/leaderelection). Managers are // generally configured to gracefully shut down controllers on pod termination // by wiring up a signal handler (pkg/manager/signals). // -// Controllers +// # Controllers // // Controllers (pkg/controller) use events (pkg/event) to eventually trigger // reconcile requests. They may be constructed manually, but are often @@ -67,7 +68,7 @@ limitations under the License. // trigger reconciles. There are pre-written utilities for the common cases, and // interfaces and helpers for advanced cases. // -// Reconcilers +// # Reconcilers // // Controller logic is implemented in terms of Reconcilers (pkg/reconcile). A // Reconciler implements a function which takes a reconcile Request containing @@ -75,7 +76,7 @@ limitations under the License. // and returns a Response or an error indicating whether to requeue for a // second round of processing. // -// Clients and Caches +// # Clients and Caches // // Reconcilers use Clients (pkg/client) to access API objects. The default // client provided by the manager reads from a local shared cache (pkg/cache) @@ -86,28 +87,28 @@ limitations under the License. // during writes (nor does it promise sequential create/get coherence), and code // should not assume a get immediately following a create/update will return // the updated resource. Caches may also have indexes, which can be created via -// a FieldIndexer (pkg/client) obtained from the manager. Indexes can used to +// a FieldIndexer (pkg/client) obtained from the manager. Indexes can be used to // quickly and easily look up all objects with certain fields set. Reconcilers // may retrieve event recorders (pkg/recorder) to emit events using the // manager. // -// Schemes +// # Schemes // // Clients, Caches, and many other things in Kubernetes use Schemes // (pkg/scheme) to associate Go types to Kubernetes API Kinds // (Group-Version-Kinds, to be specific). // -// Webhooks +// # Webhooks // // Similarly, webhooks (pkg/webhook/admission) may be implemented directly, but // are often constructed using a builder (pkg/webhook/admission/builder). They // are run via a server (pkg/webhook) which is managed by a Manager. // -// Logging and Metrics +// # Logging and Metrics // // Logging (pkg/log) in controller-runtime is done via structured logs, using a // log set of interfaces called logr -// (https://godoc.org/github.com/go-logr/logr). While controller-runtime +// (https://pkg.go.dev/github.com/go-logr/logr). While controller-runtime // provides easy setup for using Zap (https://go.uber.org/zap, pkg/log/zap), // you can provide any implementation of logr as the base logger for // controller-runtime. @@ -117,7 +118,7 @@ limitations under the License. // serve these by an HTTP endpoint, and additional metrics may be registered to // this Registry as normal. // -// Testing +// # Testing // // You can easily build integration and unit tests for your controllers and // webhooks using the test Environment (pkg/envtest). This will automatically diff --git a/example_test.go b/example_test.go index ac38733769..cbbf032b0f 100644 --- a/example_test.go +++ b/example_test.go @@ -18,14 +18,24 @@ package controllerruntime_test import ( "context" + "encoding/json" "fmt" "os" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + // since we invoke tests with -ginkgo.junit-report we need to import ginkgo. + _ "github.com/onsi/ginkgo/v2" ) // This example creates a simple application Controller that is configured for ReplicaSets and Pods. @@ -34,9 +44,8 @@ import ( // ReplicaSetReconciler. // // * Start the application. -// TODO(pwittrock): Update this example when we have better dependency injection support. func Example() { - var log = ctrl.Log.WithName("builder-examples") + log := ctrl.Log.WithName("builder-examples") manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) if err != nil { @@ -60,21 +69,108 @@ func Example() { } } +type ExampleCRDWithConfigMapRef struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + ConfigMapRef corev1.LocalObjectReference `json:"configMapRef"` +} + +func deepCopyObject(arg any) runtime.Object { + // DO NOT use this code in production code, this is only for presentation purposes. + // in real code you should generate DeepCopy methods by using controller-gen CLI tool. + argBytes, err := json.Marshal(arg) + if err != nil { + panic(err) + } + out := &ExampleCRDWithConfigMapRefList{} + if err := json.Unmarshal(argBytes, out); err != nil { + panic(err) + } + return out +} + +// DeepCopyObject implements client.Object. +func (in *ExampleCRDWithConfigMapRef) DeepCopyObject() runtime.Object { + return deepCopyObject(in) +} + +type ExampleCRDWithConfigMapRefList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ExampleCRDWithConfigMapRef `json:"items"` +} + +// DeepCopyObject implements client.ObjectList. +func (in *ExampleCRDWithConfigMapRefList) DeepCopyObject() runtime.Object { + return deepCopyObject(in) +} + +// This example creates a simple application Controller that is configured for ExampleCRDWithConfigMapRef CRD. +// Any change in the configMap referenced in this Custom Resource will cause the re-reconcile of the parent ExampleCRDWithConfigMapRef +// due to the implementation of the .Watches method of "sigs.k8s.io/controller-runtime/pkg/builder".Builder. +func Example_customHandler() { + log := ctrl.Log.WithName("builder-examples") + + manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + if err != nil { + log.Error(err, "could not create manager") + os.Exit(1) + } + + err = ctrl. + NewControllerManagedBy(manager). + For(&ExampleCRDWithConfigMapRef{}). + Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, cm client.Object) []ctrl.Request { + // map a change from referenced configMap to ExampleCRDWithConfigMapRef, which causes its re-reconcile + crList := &ExampleCRDWithConfigMapRefList{} + if err := manager.GetClient().List(ctx, crList); err != nil { + manager.GetLogger().Error(err, "while listing ExampleCRDWithConfigMapRefs") + return nil + } + + reqs := make([]ctrl.Request, 0, len(crList.Items)) + for _, item := range crList.Items { + if item.ConfigMapRef.Name == cm.GetName() { + reqs = append(reqs, ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: item.GetNamespace(), + Name: item.GetName(), + }, + }) + } + } + + return reqs + })). + Complete(reconcile.Func(func(ctx context.Context, r reconcile.Request) (reconcile.Result, error) { + // Your business logic to implement the API by creating, updating, deleting objects goes here. + return reconcile.Result{}, nil + })) + if err != nil { + log.Error(err, "could not create controller") + os.Exit(1) + } + + if err := manager.Start(ctrl.SetupSignalHandler()); err != nil { + log.Error(err, "could not start manager") + os.Exit(1) + } +} + // This example creates a simple application Controller that is configured for ReplicaSets and Pods. // This application controller will be running leader election with the provided configuration in the manager options. // If leader election configuration is not provided, controller runs leader election with default values. // Default values taken from: https://github.com/kubernetes/component-base/blob/master/config/v1alpha1/defaults.go -// defaultLeaseDuration = 15 * time.Second -// defaultRenewDeadline = 10 * time.Second -// defaultRetryPeriod = 2 * time.Second +// * defaultLeaseDuration = 15 * time.Second +// * defaultRenewDeadline = 10 * time.Second +// * defaultRetryPeriod = 2 * time.Second // // * Create a new application for ReplicaSets that manages Pods owned by the ReplicaSet and calls into // ReplicaSetReconciler. // // * Start the application. -// TODO(pwittrock): Update this example when we have better dependency injection support. func Example_updateLeaderElectionDurations() { - var log = ctrl.Log.WithName("builder-examples") + log := ctrl.Log.WithName("builder-examples") leaseDuration := 100 * time.Second renewDeadline := 80 * time.Second retryPeriod := 20 * time.Second @@ -135,7 +231,7 @@ func (a *ReplicaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Update the ReplicaSet rs.Labels["pod-count"] = fmt.Sprintf("%v", len(pods.Items)) - err = a.Update(context.TODO(), rs) + err = a.Update(ctx, rs) if err != nil { return ctrl.Result{}, err } diff --git a/examples/builtins/controller.go b/examples/builtins/controller.go index 6c8c5d935f..443283140a 100644 --- a/examples/builtins/controller.go +++ b/examples/builtins/controller.go @@ -21,7 +21,7 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -43,13 +43,13 @@ func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.R // Fetch the ReplicaSet from the cache rs := &appsv1.ReplicaSet{} err := r.client.Get(ctx, request.NamespacedName, rs) - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { log.Error(nil, "Could not find ReplicaSet") return reconcile.Result{}, nil } if err != nil { - return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+v", err) + return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+w", err) } // Print the ReplicaSet @@ -67,7 +67,7 @@ func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.R rs.Labels["hello"] = "world" err = r.client.Update(ctx, rs) if err != nil { - return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+v", err) + return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+w", err) } return reconcile.Result{}, nil diff --git a/examples/builtins/main.go b/examples/builtins/main.go index ff1f0dfa3b..3a47814d8c 100644 --- a/examples/builtins/main.go +++ b/examples/builtins/main.go @@ -22,23 +22,19 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/source" - "sigs.k8s.io/controller-runtime/pkg/webhook" ) -func init() { - log.SetLogger(zap.New()) -} - func main() { - entryLog := log.Log.WithName("entrypoint") + ctrl.SetLogger(zap.New()) + entryLog := ctrl.Log.WithName("entrypoint") // Setup a Manager entryLog.Info("setting up manager") @@ -50,35 +46,29 @@ func main() { // Setup a new controller to reconcile ReplicaSets entryLog.Info("Setting up controller") - c, err := controller.New("foo-controller", mgr, controller.Options{ - Reconciler: &reconcileReplicaSet{client: mgr.GetClient()}, - }) - if err != nil { - entryLog.Error(err, "unable to set up individual controller") - os.Exit(1) - } - // Watch ReplicaSets and enqueue ReplicaSet object key - if err := c.Watch(&source.Kind{Type: &appsv1.ReplicaSet{}}, &handler.EnqueueRequestForObject{}); err != nil { - entryLog.Error(err, "unable to watch ReplicaSets") + err = ctrl. + NewControllerManagedBy(mgr). + Named("foo-controller"). + WatchesRawSource(source.Kind(mgr.GetCache(), &appsv1.ReplicaSet{}, + &handler.TypedEnqueueRequestForObject[*appsv1.ReplicaSet]{})). + WatchesRawSource(source.Kind(mgr.GetCache(), &corev1.Pod{}, + handler.TypedEnqueueRequestForOwner[*corev1.Pod](mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()))). + Complete(&reconcileReplicaSet{client: mgr.GetClient()}) + if err != nil { + entryLog.Error(err, "could not create controller") os.Exit(1) } - // Watch Pods and enqueue owning ReplicaSet key - if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, - &handler.EnqueueRequestForOwner{OwnerType: &appsv1.ReplicaSet{}, IsController: true}); err != nil { - entryLog.Error(err, "unable to watch Pods") + if err := ctrl.NewWebhookManagedBy(mgr). + For(&corev1.Pod{}). + WithDefaulter(&podAnnotator{}). + WithValidator(&podValidator{}). + Complete(); err != nil { + entryLog.Error(err, "unable to create webhook", "webhook", "Pod") os.Exit(1) } - // Setup webhooks - entryLog.Info("setting up webhook server") - hookServer := mgr.GetWebhookServer() - - entryLog.Info("registering webhooks to the webhook server") - hookServer.Register("/mutate-v1-pod", &webhook.Admission{Handler: &podAnnotator{Client: mgr.GetClient()}}) - hookServer.Register("/validate-v1-pod", &webhook.Admission{Handler: &podValidator{Client: mgr.GetClient()}}) - entryLog.Info("starting manager") if err := mgr.Start(signals.SetupSignalHandler()); err != nil { entryLog.Error(err, "unable to run manager") diff --git a/examples/builtins/mutatingwebhook.go b/examples/builtins/mutatingwebhook.go index a4f4eee508..a588eba8f9 100644 --- a/examples/builtins/mutatingwebhook.go +++ b/examples/builtins/mutatingwebhook.go @@ -18,49 +18,31 @@ package main import ( "context" - "encoding/json" - "net/http" + "fmt" corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "k8s.io/apimachinery/pkg/runtime" + + logf "sigs.k8s.io/controller-runtime/pkg/log" ) -// +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=mpod.kb.io +// +kubebuilder:webhook:path=/mutate--v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=mpod.kb.io // podAnnotator annotates Pods -type podAnnotator struct { - Client client.Client - decoder *admission.Decoder -} +type podAnnotator struct{} -// podAnnotator adds an annotation to every incoming pods. -func (a *podAnnotator) Handle(ctx context.Context, req admission.Request) admission.Response { - pod := &corev1.Pod{} - - err := a.decoder.Decode(req, pod) - if err != nil { - return admission.Errored(http.StatusBadRequest, err) +func (a *podAnnotator) Default(ctx context.Context, obj runtime.Object) error { + log := logf.FromContext(ctx) + pod, ok := obj.(*corev1.Pod) + if !ok { + return fmt.Errorf("expected a Pod but got a %T", obj) } if pod.Annotations == nil { pod.Annotations = map[string]string{} } pod.Annotations["example-mutating-admission-webhook"] = "foo" + log.Info("Annotated Pod") - marshaledPod, err := json.Marshal(pod) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) - } - - return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) -} - -// podAnnotator implements admission.DecoderInjector. -// A decoder will be automatically injected. - -// InjectDecoder injects the decoder. -func (a *podAnnotator) InjectDecoder(d *admission.Decoder) error { - a.decoder = d return nil } diff --git a/examples/builtins/validatingwebhook.go b/examples/builtins/validatingwebhook.go index 57bc526574..1bee7f7c84 100644 --- a/examples/builtins/validatingwebhook.go +++ b/examples/builtins/validatingwebhook.go @@ -19,47 +19,48 @@ package main import ( "context" "fmt" - "net/http" corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/apimachinery/pkg/runtime" + + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -// +kubebuilder:webhook:path=/validate-v1-pod,mutating=false,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=vpod.kb.io +// +kubebuilder:webhook:path=/validate--v1-pod,mutating=false,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=vpod.kb.io // podValidator validates Pods -type podValidator struct { - Client client.Client - decoder *admission.Decoder -} - -// podValidator admits a pod if a specific annotation exists. -func (v *podValidator) Handle(ctx context.Context, req admission.Request) admission.Response { - pod := &corev1.Pod{} +type podValidator struct{} - err := v.decoder.Decode(req, pod) - if err != nil { - return admission.Errored(http.StatusBadRequest, err) +// validate admits a pod if a specific annotation exists. +func (v *podValidator) validate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + log := logf.FromContext(ctx) + pod, ok := obj.(*corev1.Pod) + if !ok { + return nil, fmt.Errorf("expected a Pod but got a %T", obj) } + log.Info("Validating Pod") key := "example-mutating-admission-webhook" anno, found := pod.Annotations[key] if !found { - return admission.Denied(fmt.Sprintf("missing annotation %s", key)) + return nil, fmt.Errorf("missing annotation %s", key) } if anno != "foo" { - return admission.Denied(fmt.Sprintf("annotation %s did not have value %q", key, "foo")) + return nil, fmt.Errorf("annotation %s did not have value %q", key, "foo") } - return admission.Allowed("") + return nil, nil } -// podValidator implements admission.DecoderInjector. -// A decoder will be automatically injected. +func (v *podValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + return v.validate(ctx, obj) +} + +func (v *podValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + return v.validate(ctx, newObj) +} -// InjectDecoder injects the decoder. -func (v *podValidator) InjectDecoder(d *admission.Decoder) error { - v.decoder = d - return nil +func (v *podValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + return v.validate(ctx, obj) } diff --git a/examples/configfile/builtin/config.yaml b/examples/configfile/builtin/config.yaml deleted file mode 100644 index 39ac86ce60..0000000000 --- a/examples/configfile/builtin/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfiguration -cacheNamespace: default -metrics: - bindAddress: :9091 -leaderElection: - leaderElect: false diff --git a/examples/configfile/builtin/controller.go b/examples/configfile/builtin/controller.go deleted file mode 100644 index 8349bcd5aa..0000000000 --- a/examples/configfile/builtin/controller.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "fmt" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -// reconcileReplicaSet reconciles ReplicaSets -type reconcileReplicaSet struct { - // client can be used to retrieve objects from the APIServer. - client client.Client -} - -// Implement reconcile.Reconciler so the controller can reconcile objects -var _ reconcile.Reconciler = &reconcileReplicaSet{} - -func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // set up a convenient log object so we don't have to type request over and over again - log := log.FromContext(ctx) - - // Fetch the ReplicaSet from the cache - rs := &appsv1.ReplicaSet{} - err := r.client.Get(context.TODO(), request.NamespacedName, rs) - if errors.IsNotFound(err) { - log.Error(nil, "Could not find ReplicaSet") - return reconcile.Result{}, nil - } - - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+v", err) - } - - // Print the ReplicaSet - log.Info("Reconciling ReplicaSet", "container name", rs.Spec.Template.Spec.Containers[0].Name) - - // Set the label if it is missing - if rs.Labels == nil { - rs.Labels = map[string]string{} - } - if rs.Labels["hello"] == "world" { - return reconcile.Result{}, nil - } - - // Update the ReplicaSet - rs.Labels["hello"] = "world" - err = r.client.Update(context.TODO(), rs) - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+v", err) - } - - return reconcile.Result{}, nil -} diff --git a/examples/configfile/builtin/main.go b/examples/configfile/builtin/main.go deleted file mode 100644 index abd6180d19..0000000000 --- a/examples/configfile/builtin/main.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client/config" - cfg "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager/signals" -) - -var scheme = runtime.NewScheme() - -func init() { - log.SetLogger(zap.New()) - clientgoscheme.AddToScheme(scheme) -} - -func main() { - entryLog := log.Log.WithName("entrypoint") - - // Setup a Manager - entryLog.Info("setting up manager") - mgr, err := ctrl.NewManager(config.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - }.AndFromOrDie(cfg.File())) - if err != nil { - entryLog.Error(err, "unable to set up overall controller manager") - os.Exit(1) - } - - // Setup a new controller to reconcile ReplicaSets - err = ctrl.NewControllerManagedBy(mgr). - For(&appsv1.ReplicaSet{}). - Owns(&corev1.Pod{}). - Complete(&reconcileReplicaSet{ - client: mgr.GetClient(), - }) - if err != nil { - entryLog.Error(err, "unable to create controller") - os.Exit(1) - } - - entryLog.Info("starting manager") - if err := mgr.Start(signals.SetupSignalHandler()); err != nil { - entryLog.Error(err, "unable to run manager") - os.Exit(1) - } -} diff --git a/examples/configfile/custom/config.yaml b/examples/configfile/custom/config.yaml deleted file mode 100644 index bf9ac044b4..0000000000 --- a/examples/configfile/custom/config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: examples.x-k8s.io/v1alpha1 -kind: CustomControllerManagerConfiguration -clusterName: example-test -cacheNamespace: default -metrics: - bindAddress: :8081 -leaderElection: - leaderElect: false diff --git a/examples/configfile/custom/controller.go b/examples/configfile/custom/controller.go deleted file mode 100644 index 8349bcd5aa..0000000000 --- a/examples/configfile/custom/controller.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "fmt" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -// reconcileReplicaSet reconciles ReplicaSets -type reconcileReplicaSet struct { - // client can be used to retrieve objects from the APIServer. - client client.Client -} - -// Implement reconcile.Reconciler so the controller can reconcile objects -var _ reconcile.Reconciler = &reconcileReplicaSet{} - -func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // set up a convenient log object so we don't have to type request over and over again - log := log.FromContext(ctx) - - // Fetch the ReplicaSet from the cache - rs := &appsv1.ReplicaSet{} - err := r.client.Get(context.TODO(), request.NamespacedName, rs) - if errors.IsNotFound(err) { - log.Error(nil, "Could not find ReplicaSet") - return reconcile.Result{}, nil - } - - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+v", err) - } - - // Print the ReplicaSet - log.Info("Reconciling ReplicaSet", "container name", rs.Spec.Template.Spec.Containers[0].Name) - - // Set the label if it is missing - if rs.Labels == nil { - rs.Labels = map[string]string{} - } - if rs.Labels["hello"] == "world" { - return reconcile.Result{}, nil - } - - // Update the ReplicaSet - rs.Labels["hello"] = "world" - err = r.client.Update(context.TODO(), rs) - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+v", err) - } - - return reconcile.Result{}, nil -} diff --git a/examples/configfile/custom/main.go b/examples/configfile/custom/main.go deleted file mode 100644 index e0fc95e337..0000000000 --- a/examples/configfile/custom/main.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/examples/configfile/custom/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client/config" - cfg "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager/signals" -) - -var scheme = runtime.NewScheme() - -func init() { - log.SetLogger(zap.New()) - clientgoscheme.AddToScheme(scheme) - v1alpha1.AddToScheme(scheme) -} - -func main() { - entryLog := log.Log.WithName("entrypoint") - - // Setup a Manager - entryLog.Info("setting up manager") - ctrlConfig := v1alpha1.CustomControllerManagerConfiguration{} - - mgr, err := ctrl.NewManager(config.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - }.AndFromOrDie(cfg.File().OfKind(&ctrlConfig))) - if err != nil { - entryLog.Error(err, "unable to set up overall controller manager") - os.Exit(1) - } - - entryLog.Info("setting up cluster", "name", ctrlConfig.ClusterName) - - // Watch ReplicaSets and enqueue ReplicaSet object key - err = ctrl.NewControllerManagedBy(mgr). - For(&appsv1.ReplicaSet{}). - Owns(&corev1.Pod{}). - Complete(&reconcileReplicaSet{ - client: mgr.GetClient(), - }) - if err != nil { - entryLog.Error(err, "unable to create controller") - os.Exit(1) - } - - entryLog.Info("starting manager") - if err := mgr.Start(signals.SetupSignalHandler()); err != nil { - entryLog.Error(err, "unable to run manager") - os.Exit(1) - } -} diff --git a/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go b/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index eff37f0a37..0000000000 --- a/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build !ignore_autogenerated - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CustomControllerManagerConfiguration) DeepCopyInto(out *CustomControllerManagerConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomControllerManagerConfiguration. -func (in *CustomControllerManagerConfiguration) DeepCopy() *CustomControllerManagerConfiguration { - if in == nil { - return nil - } - out := new(CustomControllerManagerConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *CustomControllerManagerConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} diff --git a/examples/crd/main.go b/examples/crd/main.go index 1f6cd5fac2..0bf65c9890 100644 --- a/examples/crd/main.go +++ b/examples/crd/main.go @@ -65,7 +65,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if podFound { shouldStop := chaospod.Spec.NextStop.Time.Before(time.Now()) if !shouldStop { - return ctrl.Result{RequeueAfter: chaospod.Spec.NextStop.Sub(time.Now()) + 1*time.Second}, nil + return ctrl.Result{RequeueAfter: time.Until(chaospod.Spec.NextStop.Time) + 1*time.Second}, nil } if err := r.Delete(ctx, &pod); err != nil { diff --git a/examples/crd/pkg/groupversion_info.go b/examples/crd/pkg/groupversion_info.go index 04953dd939..693d255b05 100644 --- a/examples/crd/pkg/groupversion_info.go +++ b/examples/crd/pkg/groupversion_info.go @@ -14,19 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package pkg contains API Schema definitions for the chaosapps v1 API group // +kubebuilder:object:generate=true // +groupName=chaosapps.metamagical.io package pkg import ( "k8s.io/apimachinery/pkg/runtime/schema" - logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( - log = logf.Log.WithName("chaospod-resource") - // SchemeGroupVersion is group version used to register these objects SchemeGroupVersion = schema.GroupVersion{Group: "chaosapps.metamagical.io", Version: "v1"} diff --git a/examples/crd/pkg/resource.go b/examples/crd/pkg/resource.go index 9c3d4c72bc..80800a23cb 100644 --- a/examples/crd/pkg/resource.go +++ b/examples/crd/pkg/resource.go @@ -17,13 +17,8 @@ limitations under the License. package pkg import ( - "fmt" - "time" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" ) // ChaosPodSpec defines the desired state of ChaosPod @@ -61,61 +56,6 @@ type ChaosPodList struct { Items []ChaosPod `json:"items"` } -// +kubebuilder:webhook:path=/validate-chaosapps-metamagical-io-v1-chaospod,mutating=false,failurePolicy=fail,groups=chaosapps.metamagical.io,resources=chaospods,verbs=create;update,versions=v1,name=vchaospod.kb.io - -var _ webhook.Validator = &ChaosPod{} - -// ValidateCreate implements webhookutil.validator so a webhook will be registered for the type -func (c *ChaosPod) ValidateCreate() error { - log.Info("validate create", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - return fmt.Errorf(".spec.nextStop must be later than current time") - } - return nil -} - -// ValidateUpdate implements webhookutil.validator so a webhook will be registered for the type -func (c *ChaosPod) ValidateUpdate(old runtime.Object) error { - log.Info("validate update", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - return fmt.Errorf(".spec.nextStop must be later than current time") - } - - oldC, ok := old.(*ChaosPod) - if !ok { - return fmt.Errorf("expect old object to be a %T instead of %T", oldC, old) - } - if c.Spec.NextStop.After(oldC.Spec.NextStop.Add(time.Hour)) { - return fmt.Errorf("it is not allowed to delay.spec.nextStop for more than 1 hour") - } - return nil -} - -// ValidateDelete implements webhookutil.validator so a webhook will be registered for the type -func (c *ChaosPod) ValidateDelete() error { - log.Info("validate delete", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - return fmt.Errorf(".spec.nextStop must be later than current time") - } - return nil -} - -// +kubebuilder:webhook:path=/mutate-chaosapps-metamagical-io-v1-chaospod,mutating=true,failurePolicy=fail,groups=chaosapps.metamagical.io,resources=chaospods,verbs=create;update,versions=v1,name=mchaospod.kb.io - -var _ webhook.Defaulter = &ChaosPod{} - -// Default implements webhookutil.defaulter so a webhook will be registered for the type -func (c *ChaosPod) Default() { - log.Info("default", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - c.Spec.NextStop = metav1.Time{Time: time.Now().Add(time.Minute)} - } -} - func init() { SchemeBuilder.Register(&ChaosPod{}, &ChaosPodList{}) } diff --git a/examples/multiclustersync/main.go b/examples/multiclustersync/main.go new file mode 100644 index 0000000000..e06b754222 --- /dev/null +++ b/examples/multiclustersync/main.go @@ -0,0 +1,178 @@ +package main + +import ( + "context" + "fmt" + "os" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +const ( + sourceNamespace = "namespace-to-sync-all-secrets-from" + targetNamespace = "namespace-to-sync-all-secrets-to" +) + +func run() error { + log.SetLogger(zap.New()) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + if err != nil { + return fmt.Errorf("failed to construct manager: %w", err) + } + + allTargets := map[string]cluster.Cluster{} + + cluster, err := cluster.New(ctrl.GetConfigOrDie()) + if err != nil { + return fmt.Errorf("failed to construct clusters: %w", err) + } + if err := mgr.Add(cluster); err != nil { + return fmt.Errorf("failed to add cluster to manager: %w", err) + } + + // Add more target clusters here as needed + allTargets["self"] = cluster + + b := builder.TypedControllerManagedBy[request](mgr). + Named("secret-sync"). + // Watch secrets in the source namespace of the source cluster and + // create requests for each target cluster + WatchesRawSource(source.TypedKind( + mgr.GetCache(), + &corev1.Secret{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, s *corev1.Secret) []request { + if s.Namespace != sourceNamespace { + return nil + } + + result := make([]request, 0, len(allTargets)) + for targetCluster := range allTargets { + result = append(result, request{ + NamespacedName: types.NamespacedName{Namespace: s.Namespace, Name: s.Name}, + clusterName: targetCluster, + }) + } + + return result + }), + )). + WithOptions(controller.TypedOptions[request]{MaxConcurrentReconciles: 10}) + + for targetClusterName, targetCluster := range allTargets { + // Watch secrets in the target namespace of each target cluster + // and create a request for itself. + b = b.WatchesRawSource(source.TypedKind( + targetCluster.GetCache(), + &corev1.Secret{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, s *corev1.Secret) []request { + if s.Namespace != targetNamespace { + return nil + } + + return []request{{ + NamespacedName: types.NamespacedName{Namespace: sourceNamespace, Name: s.Name}, + clusterName: targetClusterName, + }} + }), + )) + } + + clients := make(map[string]client.Client, len(allTargets)) + for targetClusterName, targetCluster := range allTargets { + clients[targetClusterName] = targetCluster.GetClient() + } + + if err := b.Complete(&secretSyncReconciler{ + source: mgr.GetClient(), + targets: clients, + }); err != nil { + return fmt.Errorf("failed to build reconciler: %w", err) + } + + ctx := signals.SetupSignalHandler() + if err := mgr.Start(ctx); err != nil { + return fmt.Errorf("failed to start manager: %w", err) + } + + return nil +} + +type request struct { + types.NamespacedName + clusterName string +} + +// secretSyncReconciler is a simple reconciler that keeps all secrets in the source namespace of a given +// source cluster in sync with the secrets in the target namespace of all target clusters. +type secretSyncReconciler struct { + source client.Client + targets map[string]client.Client +} + +func (s *secretSyncReconciler) Reconcile(ctx context.Context, req request) (reconcile.Result, error) { + targetClient, found := s.targets[req.clusterName] + if !found { + return reconcile.Result{}, reconcile.TerminalError(fmt.Errorf("target cluster %s not found", req.clusterName)) + } + + var reference corev1.Secret + if err := s.source.Get(ctx, req.NamespacedName, &reference); err != nil { + if !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to get secret %s from reference cluster: %w", req.String(), err) + } + if err := targetClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: targetNamespace, + }}); err != nil { + if !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete secret %s/%s in cluster %s: %w", targetNamespace, req.Name, req.clusterName, err) + } + + return reconcile.Result{}, nil + } + + log.FromContext(ctx).Info("Deleted secret", "cluster", req.clusterName, "namespace", targetNamespace, "name", req.Name) + return reconcile.Result{}, nil + } + + target := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: reference.Name, + Namespace: targetNamespace, + }} + result, err := controllerutil.CreateOrUpdate(ctx, targetClient, target, func() error { + target.Data = reference.Data + return nil + }) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to upsert target secret %s/%s: %w", target.Namespace, target.Name, err) + } + + if result != controllerutil.OperationResultNone { + log.FromContext(ctx).Info("Upserted secret", "cluster", req.clusterName, "namespace", targetNamespace, "name", req.Name, "result", result) + } + + return reconcile.Result{}, nil +} diff --git a/examples/priorityqueue/main.go b/examples/priorityqueue/main.go new file mode 100644 index 0000000000..1dc10c2cbe --- /dev/null +++ b/examples/priorityqueue/main.go @@ -0,0 +1,76 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "os" + "time" + + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/builder" + kubeconfig "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func init() { +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func run() error { + log.SetLogger(zap.New(func(o *zap.Options) { + o.Level = zapcore.Level(-5) + })) + + // Setup a Manager + mgr, err := manager.New(kubeconfig.GetConfigOrDie(), manager.Options{ + Controller: config.Controller{}, + }) + if err != nil { + return fmt.Errorf("failed to set up controller-manager: %w", err) + } + + if err := builder.ControllerManagedBy(mgr). + For(&corev1.ConfigMap{}). + Complete(reconcile.Func(func(ctx context.Context, r reconcile.Request) (reconcile.Result, error) { + log.FromContext(ctx).Info("Reconciling") + time.Sleep(10 * time.Second) + + return reconcile.Result{}, nil + })); err != nil { + return fmt.Errorf("failed to set up controller: %w", err) + } + + if err := mgr.Start(signals.SetupSignalHandler()); err != nil { + return fmt.Errorf("failed to start manager: %w", err) + } + + return nil +} diff --git a/examples/scratch-env/go.mod b/examples/scratch-env/go.mod index 59da6b3d9e..546c7c39ee 100644 --- a/examples/scratch-env/go.mod +++ b/examples/scratch-env/go.mod @@ -1,10 +1,70 @@ module sigs.k8s.io/controller-runtime/examples/scratch-env -go 1.15 +go 1.24.0 require ( - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6 + go.uber.org/zap v1.27.0 sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 ) +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.34.1 // indirect + k8s.io/apiextensions-apiserver v0.34.1 // indirect + k8s.io/apimachinery v0.34.1 // indirect + k8s.io/client-go v0.34.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + replace sigs.k8s.io/controller-runtime => ../.. diff --git a/examples/scratch-env/go.sum b/examples/scratch-env/go.sum index ffafe85781..012b88f447 100644 --- a/examples/scratch-env/go.sum +++ b/examples/scratch-env/go.sum @@ -1,840 +1,197 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= -github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= -github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 h1:xUIPaMhvROX9dhPvRCenIJtU78+lbEenGbgqB5hfHCQ= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c= -k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= -k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo= -k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA= -k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs= -k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= -k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= -k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4= -k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= -k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= -k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw= -k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210517184530-5a248b5acedc h1:cIS13bDBZaWqngldgGuDypv4z+zjcYgTKv72k6bMAn0= -k8s.io/utils v0.0.0-20210517184530-5a248b5acedc/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= +k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/examples/scratch-env/main.go b/examples/scratch-env/main.go index 6be4d127c2..b8305ffed3 100644 --- a/examples/scratch-env/main.go +++ b/examples/scratch-env/main.go @@ -18,14 +18,14 @@ package main import ( goflag "flag" - "io/ioutil" "os" flag "github.com/spf13/pflag" + "go.uber.org/zap" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/log/zap" + logzap "sigs.k8s.io/controller-runtime/pkg/log/zap" ) var ( @@ -36,8 +36,9 @@ var ( // have a separate function so we can return an exit code w/o skipping defers func runMain() int { - loggerOpts := &zap.Options{ + loggerOpts := &logzap.Options{ Development: true, // a sane default + ZapOpts: []zap.Option{zap.AddCaller()}, } { var goFlagSet goflag.FlagSet @@ -45,7 +46,8 @@ func runMain() int { flag.CommandLine.AddGoFlagSet(&goFlagSet) } flag.Parse() - ctrl.SetLogger(zap.New(zap.UseFlagOptions(loggerOpts))) + ctrl.SetLogger(logzap.New(logzap.UseFlagOptions(loggerOpts))) + ctrl.Log.Info("Starting...") log := ctrl.Log.WithName("main") @@ -83,7 +85,7 @@ func runMain() int { } // TODO(directxman12): add support for writing to a new context in an existing file - kubeconfigFile, err := ioutil.TempFile("", "scratch-env-kubeconfig-") + kubeconfigFile, err := os.CreateTemp("", "scratch-env-kubeconfig-") if err != nil { log.Error(err, "unable to create kubeconfig file, continuing on without it") return 1 diff --git a/examples/tokenreview/tokenreview.go b/examples/tokenreview/tokenreview.go index cc64545e16..16e4151077 100644 --- a/examples/tokenreview/tokenreview.go +++ b/examples/tokenreview/tokenreview.go @@ -28,7 +28,7 @@ import ( type authenticator struct { } -// authenticator admits a request by the token. +// Handle admits a request by the token. func (a *authenticator) Handle(ctx context.Context, req authentication.Request) authentication.Response { if req.Spec.Token == "invalid" { return authentication.Unauthenticated("invalid is an invalid token", v1.UserInfo{}) diff --git a/examples/typed/main.go b/examples/typed/main.go new file mode 100644 index 0000000000..7245ce844d --- /dev/null +++ b/examples/typed/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "fmt" + "os" + + networkingv1 "k8s.io/api/networking/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func run() error { + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + if err != nil { + return fmt.Errorf("failed to construct manager: %w", err) + } + + // Use a request type that is always equal to itself so the workqueue + // de-duplicates all events. + // This can for example be useful for an ingress-controller that + // generates a config from all ingresses, rather than individual ones. + type request struct{} + + r := reconcile.TypedFunc[request](func(ctx context.Context, _ request) (reconcile.Result, error) { + ingressList := &networkingv1.IngressList{} + if err := mgr.GetClient().List(ctx, ingressList); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to list ingresses: %w", err) + } + + buildIngressConfig(ingressList) + return reconcile.Result{}, nil + }) + if err := builder.TypedControllerManagedBy[request](mgr). + WatchesRawSource(source.TypedKind( + mgr.GetCache(), + &networkingv1.Ingress{}, + handler.TypedEnqueueRequestsFromMapFunc(func(context.Context, *networkingv1.Ingress) []request { + return []request{{}} + })), + ). + Named("ingress_controller"). + Complete(r); err != nil { + return fmt.Errorf("failed to construct ingress-controller: %w", err) + } + + ctx := signals.SetupSignalHandler() + if err := mgr.Start(ctx); err != nil { + return fmt.Errorf("failed to start manager: %w", err) + } + + return nil +} + +func buildIngressConfig(*networkingv1.IngressList) {} diff --git a/go.mod b/go.mod index fabf9b8f66..4d998fe2fc 100644 --- a/go.mod +++ b/go.mod @@ -1,28 +1,103 @@ module sigs.k8s.io/controller-runtime -go 1.16 +go 1.24.0 require ( - github.com/evanphx/json-patch v4.11.0+incompatible - github.com/fsnotify/fsnotify v1.4.9 - github.com/go-logr/logr v0.4.0 - github.com/go-logr/zapr v0.4.0 - github.com/imdario/mergo v0.3.12 // indirect - github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.15.0 - github.com/prometheus/client_golang v1.11.0 - github.com/prometheus/client_model v0.2.0 - go.uber.org/goleak v1.1.10 - go.uber.org/zap v1.19.0 - golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac - gomodules.xyz/jsonpatch/v2 v2.2.0 - google.golang.org/appengine v1.6.7 // indirect - k8s.io/api v0.22.2 - k8s.io/apiextensions-apiserver v0.22.2 - k8s.io/apimachinery v0.22.2 - k8s.io/client-go v0.22.2 - k8s.io/component-base v0.22.2 - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a - sigs.k8s.io/yaml v1.2.0 + github.com/evanphx/json-patch/v5 v5.9.11 + github.com/fsnotify/fsnotify v1.9.0 + github.com/go-logr/logr v1.4.2 + github.com/go-logr/zapr v1.3.0 + github.com/google/btree v1.1.3 + github.com/google/go-cmp v0.7.0 + github.com/google/gofuzz v1.2.0 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + github.com/prometheus/client_golang v1.22.0 + github.com/prometheus/client_model v0.6.1 + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 + golang.org/x/mod v0.21.0 + golang.org/x/sync v0.12.0 + golang.org/x/sys v0.31.0 + gomodules.xyz/jsonpatch/v2 v2.4.0 + gopkg.in/evanphx/json-patch.v4 v4.12.0 // Using v4 to match upstream + k8s.io/api v0.34.1 + k8s.io/apiextensions-apiserver v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/apiserver v0.34.1 + k8s.io/client-go v0.34.1 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + cel.dev/expr v0.24.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/grpc v1.72.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/component-base v0.34.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 293f15d330..d6278d8a7d 100644 --- a/go.sum +++ b/go.sum @@ -1,769 +1,259 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= -k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= -k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= -k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= -k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= -k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= -k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= -k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= -k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= -k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= +k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= +k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= +k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/apidiff.sh b/hack/apidiff.sh index 0167486da1..a15342d16a 100755 --- a/hack/apidiff.sh +++ b/hack/apidiff.sh @@ -23,13 +23,8 @@ source $(dirname ${BASH_SOURCE})/common.sh REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. cd "${REPO_ROOT}" -APIDIFF="hack/tools/bin/go-apidiff" - -header_text "fetching and building go-apidiff" -make "${APIDIFF}" - -git status +export GOTOOLCHAIN="go$(make --silent go-version)" header_text "verifying api diff" -header_text "invoking: '${APIDIFF} ${PULL_BASE_SHA} --print-compatible'" -"${APIDIFF}" "${PULL_BASE_SHA}" --print-compatible +echo "*** Running go-apidiff ***" +APIDIFF_OLD_COMMIT="${PULL_BASE_SHA}" make verify-apidiff diff --git a/hack/check-everything.sh b/hack/check-everything.sh index de559643cc..84db032176 100755 --- a/hack/check-everything.sh +++ b/hack/check-everything.sh @@ -24,11 +24,13 @@ source ${hack_dir}/common.sh tmp_root=/tmp kb_root_dir=$tmp_root/kubebuilder +export GOTOOLCHAIN="go$(make --silent go-version)" + # Run verification scripts. ${hack_dir}/verify.sh # Envtest. -ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.22.0"} +ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.32.0"} header_text "installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest if necessary" tmp_bin=/tmp/cr-tests-bin @@ -45,8 +47,6 @@ ${hack_dir}/test-all.sh header_text "confirming examples compile (via go install)" go install ${MOD_OPT} ./examples/builtins go install ${MOD_OPT} ./examples/crd -go install ${MOD_OPT} ./examples/configfile/builtin -go install ${MOD_OPT} ./examples/configfile/custom echo "passed" exit 0 diff --git a/hack/ensure-golangci-lint.sh b/hack/ensure-golangci-lint.sh deleted file mode 100755 index 9e9ef03167..0000000000 --- a/hack/ensure-golangci-lint.sh +++ /dev/null @@ -1,422 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: This script is copied from from https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh. - -set -e - -usage() { - this=$1 - cat </dev/null -} -echoerr() { - echo "$@" 1>&2 -} -log_prefix() { - echo "$0" -} -_logp=6 -log_set_priority() { - _logp="$1" -} -log_priority() { - if test -z "$1"; then - echo "$_logp" - return - fi - [ "$1" -le "$_logp" ] -} -log_tag() { - case $1 in - 0) echo "emerg" ;; - 1) echo "alert" ;; - 2) echo "crit" ;; - 3) echo "err" ;; - 4) echo "warning" ;; - 5) echo "notice" ;; - 6) echo "info" ;; - 7) echo "debug" ;; - *) echo "$1" ;; - esac -} -log_debug() { - log_priority 7 || return 0 - echoerr "$(log_prefix)" "$(log_tag 7)" "$@" -} -log_info() { - log_priority 6 || return 0 - echoerr "$(log_prefix)" "$(log_tag 6)" "$@" -} -log_err() { - log_priority 3 || return 0 - echoerr "$(log_prefix)" "$(log_tag 3)" "$@" -} -log_crit() { - log_priority 2 || return 0 - echoerr "$(log_prefix)" "$(log_tag 2)" "$@" -} -uname_os() { - os=$(uname -s | tr '[:upper:]' '[:lower:]') - case "$os" in - cygwin_nt*) os="windows" ;; - mingw*) os="windows" ;; - msys_nt*) os="windows" ;; - esac - echo "$os" -} -uname_arch() { - arch=$(uname -m) - case $arch in - x86_64) arch="amd64" ;; - x86) arch="386" ;; - i686) arch="386" ;; - i386) arch="386" ;; - aarch64) arch="arm64" ;; - armv5*) arch="armv5" ;; - armv6*) arch="armv6" ;; - armv7*) arch="armv7" ;; - esac - echo ${arch} -} -uname_os_check() { - os=$(uname_os) - case "$os" in - darwin) return 0 ;; - dragonfly) return 0 ;; - freebsd) return 0 ;; - linux) return 0 ;; - android) return 0 ;; - nacl) return 0 ;; - netbsd) return 0 ;; - openbsd) return 0 ;; - plan9) return 0 ;; - solaris) return 0 ;; - windows) return 0 ;; - esac - log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" - return 1 -} -uname_arch_check() { - arch=$(uname_arch) - case "$arch" in - 386) return 0 ;; - amd64) return 0 ;; - arm64) return 0 ;; - armv5) return 0 ;; - armv6) return 0 ;; - armv7) return 0 ;; - ppc64) return 0 ;; - ppc64le) return 0 ;; - mips) return 0 ;; - mipsle) return 0 ;; - mips64) return 0 ;; - mips64le) return 0 ;; - s390x) return 0 ;; - amd64p32) return 0 ;; - esac - log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" - return 1 -} -untar() { - tarball=$1 - case "${tarball}" in - *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;; - *.tar) tar --no-same-owner -xf "${tarball}" ;; - *.zip) unzip "${tarball}" ;; - *) - log_err "untar unknown archive format for ${tarball}" - return 1 - ;; - esac -} -http_download_curl() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") - else - code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") - fi - if [ "$code" != "200" ]; then - log_debug "http_download_curl received HTTP status $code" - return 1 - fi - return 0 -} -http_download_wget() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - wget -q -O "$local_file" "$source_url" - else - wget -q --header "$header" -O "$local_file" "$source_url" - fi -} -http_download() { - log_debug "http_download $2" - if is_command curl; then - http_download_curl "$@" - return - elif is_command wget; then - http_download_wget "$@" - return - fi - log_crit "http_download unable to find wget or curl" - return 1 -} -http_copy() { - tmp=$(mktemp) - http_download "${tmp}" "$1" "$2" || return 1 - body=$(cat "$tmp") - rm -f "${tmp}" - echo "$body" -} -github_release() { - owner_repo=$1 - version=$2 - test -z "$version" && version="latest" - giturl="https://github.com/${owner_repo}/releases/${version}" - json=$(http_copy "$giturl" "Accept:application/json") - test -z "$json" && return 1 - version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') - test -z "$version" && return 1 - echo "$version" -} -hash_sha256() { - TARGET=${1:-/dev/stdin} - if is_command gsha256sum; then - hash=$(gsha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command sha256sum; then - hash=$(sha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command shasum; then - hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command openssl; then - hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f a - else - log_crit "hash_sha256 unable to find command to compute sha-256 hash" - return 1 - fi -} -hash_sha256_verify() { - TARGET=$1 - checksums=$2 - if [ -z "$checksums" ]; then - log_err "hash_sha256_verify checksum file not specified in arg2" - return 1 - fi - BASENAME=${TARGET##*/} - want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) - if [ -z "$want" ]; then - log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" - return 1 - fi - got=$(hash_sha256 "$TARGET") - if [ "$want" != "$got" ]; then - log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" - return 1 - fi -} -cat /dev/null <' ${ARTIFACTS}/*; then exit 1; fi + mkdir -p ${ARTIFACTS} + for file in `find . -name *junit-report.xml`; do + new_file=${file#./} + new_file=${new_file%/junit-report.xml} + new_file=${new_file//"/"/"-"} + mv "$file" "$ARTIFACTS/junit_${new_file}.xml" + done fi + +exit $result diff --git a/hack/tools/cmd/gomodcheck/main.go b/hack/tools/cmd/gomodcheck/main.go new file mode 100644 index 0000000000..5cbaf377e2 --- /dev/null +++ b/hack/tools/cmd/gomodcheck/main.go @@ -0,0 +1,204 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + + "go.uber.org/zap" + "golang.org/x/mod/modfile" + "sigs.k8s.io/yaml" +) + +const ( + modFile = "./go.mod" +) + +type config struct { + UpstreamRefs []string `json:"upstreamRefs"` + ExcludedModules []string `json:"excludedModules"` +} + +type upstream struct { + Ref string `json:"ref"` + Version string `json:"version"` +} + +// representation of an out of sync module +type oosMod struct { + Name string `json:"name"` + Version string `json:"version"` + Upstreams []upstream `json:"upstreams"` +} + +func main() { + l, _ := zap.NewProduction() + logger := l.Sugar() + + if len(os.Args) < 2 { + fmt.Printf("USAGE: %s [PATH_TO_CONFIG_FILE]\n", os.Args[0]) + os.Exit(1) + } + + // --- 1. parse config + b, err := os.ReadFile(os.Args[1]) + if err != nil { + fatal(err) + } + + cfg := new(config) + if err := yaml.Unmarshal(b, cfg); err != nil { + fatal(err) + } + + excludedMods := make(map[string]any) + for _, mod := range cfg.ExcludedModules { + excludedMods[mod] = nil + } + + // --- 2. project mods + projectModules, err := modulesFromGoModFile() + if err != nil { + fatal(err) + } + + // --- 3. upstream mods + upstreamModules, err := modulesFromUpstreamModGraph(cfg.UpstreamRefs) + if err != nil { + fatal(err) + } + + oosMods := make([]oosMod, 0) + + // --- 4. validate + // for each module in our project, + // if it matches an upstream module, + // then for each upstream module, + // if project module version doesn't match upstream version, + // then we add the version and the ref to the list of out of sync modules. + for mod, version := range projectModules { + if _, ok := excludedMods[mod]; ok { + logger.Infof("skipped: %s", mod) + continue + } + + if versionToRef, ok := upstreamModules[mod]; ok { + outOfSyncUpstream := make([]upstream, 0) + + for upstreamVersion, upstreamRef := range versionToRef { + if version == upstreamVersion { // pass if version in sync. + continue + } + + outOfSyncUpstream = append(outOfSyncUpstream, upstream{ + Ref: upstreamRef, + Version: upstreamVersion, + }) + } + + if len(outOfSyncUpstream) == 0 { // pass if no out of sync upstreams. + continue + } + + oosMods = append(oosMods, oosMod{ + Name: mod, + Version: version, + Upstreams: outOfSyncUpstream, + }) + } + } + + if len(oosMods) == 0 { + fmt.Println("🎉 Success!") + os.Exit(0) + } + + b, err = json.MarshalIndent(map[string]any{"outOfSyncModules": oosMods}, "", " ") + if err != nil { + fatal(err) + } + + fmt.Println(string(b)) + os.Exit(1) +} + +func modulesFromGoModFile() (map[string]string, error) { + b, err := os.ReadFile(modFile) + if err != nil { + return nil, err + } + + f, err := modfile.Parse(modFile, b, nil) + if err != nil { + return nil, err + } + + out := make(map[string]string) + for _, mod := range f.Require { + out[mod.Mod.Path] = mod.Mod.Version + } + + return out, nil +} + +func modulesFromUpstreamModGraph(upstreamRefList []string) (map[string]map[string]string, error) { + b, err := exec.Command("go", "mod", "graph").Output() + if err != nil { + return nil, err + } + + graph := string(b) + + // upstreamRefs is a set of user specified upstream modules. + // The set has 2 functions: + // 1. Check if `go mod graph` modules are one of the user specified upstream modules. + // 2. Mark if a user specified upstream module was found in the module graph. + // If a user specified upstream module is not found, gomodcheck will exit with an error. + upstreamRefs := make(map[string]bool) + for _, ref := range upstreamRefList { + upstreamRefs[ref] = false + } + + modToVersionToUpstreamRef := make(map[string]map[string]string) + for _, line := range strings.Split(graph, "\n") { + ref := strings.SplitN(line, "@", 2)[0] + + if _, ok := upstreamRefs[ref]; !ok { + continue + } + + upstreamRefs[ref] = true // mark the ref as found + + kv := strings.SplitN(strings.SplitN(line, " ", 2)[1], "@", 2) + name := kv[0] + version := kv[1] + + if _, ok := modToVersionToUpstreamRef[name]; !ok { + modToVersionToUpstreamRef[name] = make(map[string]string) + } + + modToVersionToUpstreamRef[name][version] = ref + } + + notFoundErr := "" + for ref, found := range upstreamRefs { + if !found { + notFoundErr = fmt.Sprintf("%s%s, ", notFoundErr, ref) + } + } + + if notFoundErr != "" { + return nil, fmt.Errorf("cannot verify modules: "+ + "the following specified upstream module(s) cannot be found in go.mod: [ %s ]", + strings.TrimSuffix(notFoundErr, ", ")) + } + + return modToVersionToUpstreamRef, nil +} + +func fatal(err error) { + fmt.Printf("❌ %s\n", err.Error()) + os.Exit(1) +} diff --git a/hack/tools/go.mod b/hack/tools/go.mod deleted file mode 100644 index 78a4939421..0000000000 --- a/hack/tools/go.mod +++ /dev/null @@ -1,8 +0,0 @@ -module sigs.k8s.io/controller-runtime/hack/tools - -go 1.16 - -require ( - github.com/joelanford/go-apidiff v0.1.0 - sigs.k8s.io/controller-tools v0.6.2 -) diff --git a/hack/tools/go.sum b/hack/tools/go.sum deleted file mode 100644 index 0cdb2d1bc6..0000000000 --- a/hack/tools/go.sum +++ /dev/null @@ -1,929 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobuffalo/flect v0.2.3 h1:f/ZukRnSNA/DUpSNDadko7Qc0PhGvsew35p/2tu+CRY= -github.com/gobuffalo/flect v0.2.3/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/joelanford/go-apidiff v0.1.0 h1:bt/247wfLDKFnCC5jYdapR3WY2laJMPB9apfc1U9Idw= -github.com/joelanford/go-apidiff v0.1.0/go.mod h1:wgVWgVCwYYkjcYpJtBnWYkyUYZfVovO3Y5pX49mJsqs= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI= -github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191004183538-27eeabb02079/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.3 h1:cblWILbLO8ar+Fj6xdDGr603HRsf8Wu9E9rngJeprZQ= -k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= -k8s.io/apiextensions-apiserver v0.21.3 h1:+B6biyUWpqt41kz5x6peIsljlsuwvNAp/oFax/j2/aY= -k8s.io/apiextensions-apiserver v0.21.3/go.mod h1:kl6dap3Gd45+21Jnh6utCx8Z2xxLm8LGDkprcd+KbsE= -k8s.io/apimachinery v0.21.3 h1:3Ju4nvjCngxxMYby0BimUk+pQHPOQp3eCGChk5kfVII= -k8s.io/apimachinery v0.21.3/go.mod h1:H/IM+5vH9kZRNJ4l3x/fXP/5bOPJaVP/guptnZPeCFI= -k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= -k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= -k8s.io/code-generator v0.21.3/go.mod h1:K3y0Bv9Cz2cOW2vXUrNZlFbflhuPvuadW6JdnN6gGKo= -k8s.io/component-base v0.21.3/go.mod h1:kkuhtfEHeZM6LkX0saqSK8PbdO7A0HigUngmhhrwfGQ= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-tools v0.6.2 h1:+Y8L0UsAugDipGRw8lrkPoAi6XqlQVZuf1DQHME3PgU= -sigs.k8s.io/controller-tools v0.6.2/go.mod h1:oaeGpjXn6+ZSEIQkUe/+3I40PNiDYp9aeawbt3xTgJ8= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/hack/verify-pr-title.sh b/hack/verify-pr-title.sh new file mode 100755 index 0000000000..a556b0172b --- /dev/null +++ b/hack/verify-pr-title.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define regex patterns +WIP_REGEX="^\W?WIP\W" +TAG_REGEX="^\[[[:alnum:]\._-]*\]" +PR_TITLE="$1" + +# Trim WIP and tags from title +trimmed_title=$(echo "$PR_TITLE" | sed -E "s/$WIP_REGEX//" | sed -E "s/$TAG_REGEX//" | xargs) + +# Normalize common emojis in text form to actual emojis +trimmed_title=$(echo "$trimmed_title" | sed -E "s/:warning:/⚠/g") +trimmed_title=$(echo "$trimmed_title" | sed -E "s/:sparkles:/✨/g") +trimmed_title=$(echo "$trimmed_title" | sed -E "s/:bug:/🐛/g") +trimmed_title=$(echo "$trimmed_title" | sed -E "s/:book:/📖/g") +trimmed_title=$(echo "$trimmed_title" | sed -E "s/:rocket:/🚀/g") +trimmed_title=$(echo "$trimmed_title" | sed -E "s/:seedling:/🌱/g") + +# Check PR type prefix +if [[ "$trimmed_title" =~ ^(⚠|✨|🐛|📖|🚀|🌱) ]]; then + echo "PR title is valid: $trimmed_title" +else + echo "Error: No matching PR type indicator found in title." + echo "You need to have one of these as the prefix of your PR title:" + echo "- Breaking change: ⚠ (:warning:)" + echo "- Non-breaking feature: ✨ (:sparkles:)" + echo "- Patch fix: 🐛 (:bug:)" + echo "- Docs: 📖 (:book:)" + echo "- Release: 🚀 (:rocket:)" + echo "- Infra/Tests/Other: 🌱 (:seedling:)" + exit 1 +fi + +# Check that PR title does not contain Issue or PR number +if [[ "$trimmed_title" =~ \#[0-9]+ ]]; then + echo "Error: PR title should not contain issue or PR number." + echo "Issue numbers belong in the PR body as either \"Fixes #XYZ\" (if it closes the issue or PR), or something like \"Related to #XYZ\" (if it's just related)." + exit 1 +fi + diff --git a/hack/verify.sh b/hack/verify.sh index 85006e3f06..4ce46d15d1 100755 --- a/hack/verify.sh +++ b/hack/verify.sh @@ -21,15 +21,15 @@ source $(dirname ${BASH_SOURCE})/common.sh REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. cd "${REPO_ROOT}" -header_text "running generate" -make generate +header_text "running modules" +make modules -header_text "running golangci-lint" -make lint - -# Only run module verification in CI, otherwise updating +# Only run verify-modules in CI, otherwise updating # go module locally (which is a valid operation) causes `make test` to fail. if [[ -n ${CI} ]]; then header_text "verifying modules" - make modules verify-modules + make verify-modules fi + +header_text "running golangci-lint" +make lint diff --git a/pkg/builder/builder_suite_test.go b/pkg/builder/builder_suite_test.go index 5ae6fd8616..11f5b8684d 100644 --- a/pkg/builder/builder_suite_test.go +++ b/pkg/builder/builder_suite_test.go @@ -19,27 +19,25 @@ package builder import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/webhook" ) func TestBuilder(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "application Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "application Suite") } var testenv *envtest.Environment @@ -59,17 +57,17 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) // Prevent the metrics listener being created - metrics.DefaultBindAddress = "0" + metricsserver.DefaultBindAddress = "0" webhook.DefaultPort, _, err = addr.Suggest("") Expect(err).NotTo(HaveOccurred()) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) // Put the DefaultBindAddress back - metrics.DefaultBindAddress = ":8080" + metricsserver.DefaultBindAddress = ":8080" // Change the webhook.DefaultPort back to the original default. webhook.DefaultPort = 9443 diff --git a/pkg/builder/controller.go b/pkg/builder/controller.go index 2cd4ce9de1..6d906f6e52 100644 --- a/pkg/builder/controller.go +++ b/pkg/builder/controller.go @@ -17,12 +17,15 @@ limitations under the License. package builder import ( + "errors" "fmt" + "reflect" "strings" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -34,39 +37,47 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) -// Supporting mocking out functions for testing. -var newController = controller.New -var getGvk = apiutil.GVKForObject - -// project represents other forms that the we can use to +// project represents other forms that we can use to // send/receive a given resource (metadata-only, unstructured, etc). type objectProjection int const ( // projectAsNormal doesn't change the object from the form given. projectAsNormal objectProjection = iota - // projectAsMetadata turns this into an metadata-only watch. + // projectAsMetadata turns this into a metadata-only watch. projectAsMetadata ) // Builder builds a Controller. -type Builder struct { +type Builder = TypedBuilder[reconcile.Request] + +// TypedBuilder builds a Controller. The request is the request type +// that is passed to the workqueue and then to the Reconciler. +// The workqueue de-duplicates identical requests. +type TypedBuilder[request comparable] struct { forInput ForInput ownsInput []OwnsInput - watchesInput []WatchesInput + rawSources []source.TypedSource[request] + watchesInput []WatchesInput[request] mgr manager.Manager globalPredicates []predicate.Predicate - ctrl controller.Controller - ctrlOptions controller.Options + ctrl controller.TypedController[request] + ctrlOptions controller.TypedOptions[request] name string + newController func(name string, mgr manager.Manager, options controller.TypedOptions[request]) (controller.TypedController[request], error) } // ControllerManagedBy returns a new controller builder that will be started by the provided Manager. func ControllerManagedBy(m manager.Manager) *Builder { - return &Builder{mgr: m} + return TypedControllerManagedBy[reconcile.Request](m) } -// ForInput represents the information set by For method. +// TypedControllerManagedBy returns a new typed controller builder that will be started by the provided Manager. +func TypedControllerManagedBy[request comparable](m manager.Manager) *TypedBuilder[request] { + return &TypedBuilder[request]{mgr: m} +} + +// ForInput represents the information set by the For method. type ForInput struct { object client.Object predicates []predicate.Predicate @@ -76,9 +87,10 @@ type ForInput struct { // For defines the type of Object being *reconciled*, and configures the ControllerManagedBy to respond to create / delete / // update events by *reconciling the object*. +// // This is the equivalent of calling -// Watches(&source.Kind{Type: apiType}, &handler.EnqueueRequestForObject{}). -func (blder *Builder) For(object client.Object, opts ...ForOption) *Builder { +// Watches(source.Kind(cache, &Type{}, &handler.EnqueueRequestForObject{})). +func (blder *TypedBuilder[request]) For(object client.Object, opts ...ForOption) *TypedBuilder[request] { if blder.forInput.object != nil { blder.forInput.err = fmt.Errorf("For(...) should only be called once, could not assign multiple objects for reconciliation") return blder @@ -94,15 +106,21 @@ func (blder *Builder) For(object client.Object, opts ...ForOption) *Builder { // OwnsInput represents the information set by Owns method. type OwnsInput struct { + matchEveryOwner bool object client.Object predicates []predicate.Predicate objectProjection objectProjection } // Owns defines types of Objects being *generated* by the ControllerManagedBy, and configures the ControllerManagedBy to respond to -// create / delete / update events by *reconciling the owner object*. This is the equivalent of calling -// Watches(&source.Kind{Type: }, &handler.EnqueueRequestForOwner{OwnerType: apiType, IsController: true}). -func (blder *Builder) Owns(object client.Object, opts ...OwnsOption) *Builder { +// create / delete / update events by *reconciling the owner object*. +// +// The default behavior reconciles only the first controller-type OwnerReference of the given type. +// Use Owns(object, builder.MatchEveryOwner) to reconcile all owners. +// +// By default, this is the equivalent of calling +// Watches(source.Kind(cache, &Type{}, handler.EnqueueRequestForOwner([...], &OwnerType{}, OnlyControllerOwner()))). +func (blder *TypedBuilder[request]) Owns(object client.Object, opts ...OwnsOption) *TypedBuilder[request] { input := OwnsInput{object: object} for _, opt := range opts { opt.ApplyToOwns(&input) @@ -112,66 +130,140 @@ func (blder *Builder) Owns(object client.Object, opts ...OwnsOption) *Builder { return blder } +type untypedWatchesInput interface { + setPredicates([]predicate.Predicate) + setObjectProjection(objectProjection) +} + // WatchesInput represents the information set by Watches method. -type WatchesInput struct { - src source.Source - eventhandler handler.EventHandler +type WatchesInput[request comparable] struct { + obj client.Object + handler handler.TypedEventHandler[client.Object, request] predicates []predicate.Predicate objectProjection objectProjection } -// Watches exposes the lower-level ControllerManagedBy Watches functions through the builder. Consider using -// Owns or For instead of Watches directly. -// Specified predicates are registered only for given source. -func (blder *Builder) Watches(src source.Source, eventhandler handler.EventHandler, opts ...WatchesOption) *Builder { - input := WatchesInput{src: src, eventhandler: eventhandler} +func (w *WatchesInput[request]) setPredicates(predicates []predicate.Predicate) { + w.predicates = predicates +} + +func (w *WatchesInput[request]) setObjectProjection(objectProjection objectProjection) { + w.objectProjection = objectProjection +} + +// Watches defines the type of Object to watch, and configures the ControllerManagedBy to respond to create / delete / +// update events by *reconciling the object* with the given EventHandler. +// +// This is the equivalent of calling +// WatchesRawSource(source.Kind(cache, object, eventHandler, predicates...)). +func (blder *TypedBuilder[request]) Watches( + object client.Object, + eventHandler handler.TypedEventHandler[client.Object, request], + opts ...WatchesOption, +) *TypedBuilder[request] { + input := WatchesInput[request]{ + obj: object, + handler: eventHandler, + } for _, opt := range opts { opt.ApplyToWatches(&input) } blder.watchesInput = append(blder.watchesInput, input) + + return blder +} + +// WatchesMetadata is the same as Watches, but forces the internal cache to only watch PartialObjectMetadata. +// +// This is useful when watching lots of objects, really big objects, or objects for which you only know +// the GVK, but not the structure. You'll need to pass metav1.PartialObjectMetadata to the client +// when fetching objects in your reconciler, otherwise you'll end up with a duplicate structured or unstructured cache. +// +// When watching a resource with metadata only, for example the v1.Pod, you should not Get and List using the v1.Pod type. +// Instead, you should use the special metav1.PartialObjectMetadata type. +// +// ❌ Incorrect: +// +// pod := &v1.Pod{} +// mgr.GetClient().Get(ctx, nsAndName, pod) +// +// ✅ Correct: +// +// pod := &metav1.PartialObjectMetadata{} +// pod.SetGroupVersionKind(schema.GroupVersionKind{ +// Group: "", +// Version: "v1", +// Kind: "Pod", +// }) +// mgr.GetClient().Get(ctx, nsAndName, pod) +// +// In the first case, controller-runtime will create another cache for the +// concrete type on top of the metadata cache; this increases memory +// consumption and leads to race conditions as caches are not in sync. +func (blder *TypedBuilder[request]) WatchesMetadata( + object client.Object, + eventHandler handler.TypedEventHandler[client.Object, request], + opts ...WatchesOption, +) *TypedBuilder[request] { + opts = append(opts, OnlyMetadata) + return blder.Watches(object, eventHandler, opts...) +} + +// WatchesRawSource exposes the lower-level ControllerManagedBy Watches functions through the builder. +// +// WatchesRawSource does not respect predicates configured through WithEventFilter. +// +// WatchesRawSource makes it possible to use typed handlers and predicates with `source.Kind` as well as custom source implementations. +func (blder *TypedBuilder[request]) WatchesRawSource(src source.TypedSource[request]) *TypedBuilder[request] { + blder.rawSources = append(blder.rawSources, src) + return blder } // WithEventFilter sets the event filters, to filter which create/update/delete/generic events eventually -// trigger reconciliations. For example, filtering on whether the resource version has changed. -// Given predicate is added for all watched objects. +// trigger reconciliations. For example, filtering on whether the resource version has changed. +// Given predicate is added for all watched objects and thus must be able to deal with the type +// of all watched objects. +// // Defaults to the empty list. -func (blder *Builder) WithEventFilter(p predicate.Predicate) *Builder { +func (blder *TypedBuilder[request]) WithEventFilter(p predicate.Predicate) *TypedBuilder[request] { blder.globalPredicates = append(blder.globalPredicates, p) return blder } -// WithOptions overrides the controller options use in doController. Defaults to empty. -func (blder *Builder) WithOptions(options controller.Options) *Builder { +// WithOptions overrides the controller options used in doController. Defaults to empty. +func (blder *TypedBuilder[request]) WithOptions(options controller.TypedOptions[request]) *TypedBuilder[request] { blder.ctrlOptions = options return blder } -// WithLogger overrides the controller options's logger used. -func (blder *Builder) WithLogger(log logr.Logger) *Builder { - blder.ctrlOptions.Log = log +// WithLogConstructor overrides the controller options's LogConstructor. +func (blder *TypedBuilder[request]) WithLogConstructor(logConstructor func(*request) logr.Logger) *TypedBuilder[request] { + blder.ctrlOptions.LogConstructor = logConstructor return blder } -// Named sets the name of the controller to the given name. The name shows up +// Named sets the name of the controller to the given name. The name shows up // in metrics, among other things, and thus should be a prometheus compatible name // (underscores and alphanumeric characters only). // // By default, controllers are named using the lowercase version of their kind. -func (blder *Builder) Named(name string) *Builder { +// +// The name must be unique as it is used to identify the controller in metrics and logs. +func (blder *TypedBuilder[request]) Named(name string) *TypedBuilder[request] { blder.name = name return blder } // Complete builds the Application Controller. -func (blder *Builder) Complete(r reconcile.Reconciler) error { +func (blder *TypedBuilder[request]) Complete(r reconcile.TypedReconciler[request]) error { _, err := blder.Build(r) return err } // Build builds the Application Controller and returns the Controller it created. -func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, error) { +func (blder *TypedBuilder[request]) Build(r reconcile.TypedReconciler[request]) (controller.TypedController[request], error) { if r == nil { return nil, fmt.Errorf("must provide a non-nil Reconciler") } @@ -181,10 +273,6 @@ func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, erro if blder.forInput.err != nil { return nil, blder.forInput.err } - // Checking the reconcile type exist or not - if blder.forInput.object == nil { - return nil, fmt.Errorf("must provide an object for reconciliation") - } // Set the ControllerManagedBy if err := blder.doController(r); err != nil { @@ -199,13 +287,13 @@ func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, erro return blder.ctrl, nil } -func (blder *Builder) project(obj client.Object, proj objectProjection) (client.Object, error) { +func (blder *TypedBuilder[request]) project(obj client.Object, proj objectProjection) (client.Object, error) { switch proj { case projectAsNormal: return obj, nil case projectAsMetadata: metaObj := &metav1.PartialObjectMetadata{} - gvk, err := getGvk(obj, blder.mgr.GetScheme()) + gvk, err := apiutil.GVKForObject(obj, blder.mgr.GetScheme()) if err != nil { return nil, fmt.Errorf("unable to determine GVK of %T for a metadata-only watch: %w", obj, err) } @@ -216,82 +304,114 @@ func (blder *Builder) project(obj client.Object, proj objectProjection) (client. } } -func (blder *Builder) doWatch() error { +func (blder *TypedBuilder[request]) doWatch() error { // Reconcile type - typeForSrc, err := blder.project(blder.forInput.object, blder.forInput.objectProjection) - if err != nil { - return err - } - src := &source.Kind{Type: typeForSrc} - hdler := &handler.EnqueueRequestForObject{} - allPredicates := append(blder.globalPredicates, blder.forInput.predicates...) - if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil { - return err + if blder.forInput.object != nil { + obj, err := blder.project(blder.forInput.object, blder.forInput.objectProjection) + if err != nil { + return err + } + + if reflect.TypeFor[request]() != reflect.TypeOf(reconcile.Request{}) { + return fmt.Errorf("For() can only be used with reconcile.Request, got %T", *new(request)) + } + + var hdler handler.TypedEventHandler[client.Object, request] + reflect.ValueOf(&hdler).Elem().Set(reflect.ValueOf(&handler.EnqueueRequestForObject{})) + allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) + allPredicates = append(allPredicates, blder.forInput.predicates...) + src := source.TypedKind(blder.mgr.GetCache(), obj, hdler, allPredicates...) + if err := blder.ctrl.Watch(src); err != nil { + return err + } } // Watches the managed types + if len(blder.ownsInput) > 0 && blder.forInput.object == nil { + return errors.New("Owns() can only be used together with For()") + } for _, own := range blder.ownsInput { - typeForSrc, err := blder.project(own.object, own.objectProjection) + obj, err := blder.project(own.object, own.objectProjection) if err != nil { return err } - src := &source.Kind{Type: typeForSrc} - hdler := &handler.EnqueueRequestForOwner{ - OwnerType: blder.forInput.object, - IsController: true, + opts := []handler.OwnerOption{} + if !own.matchEveryOwner { + opts = append(opts, handler.OnlyControllerOwner()) } + + var hdler handler.TypedEventHandler[client.Object, request] + reflect.ValueOf(&hdler).Elem().Set(reflect.ValueOf(handler.EnqueueRequestForOwner( + blder.mgr.GetScheme(), blder.mgr.GetRESTMapper(), + blder.forInput.object, + opts..., + ))) allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) allPredicates = append(allPredicates, own.predicates...) - if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil { + src := source.TypedKind(blder.mgr.GetCache(), obj, hdler, allPredicates...) + if err := blder.ctrl.Watch(src); err != nil { return err } } // Do the watch requests + if len(blder.watchesInput) == 0 && blder.forInput.object == nil && len(blder.rawSources) == 0 { + return errors.New("there are no watches configured, controller will never get triggered. Use For(), Owns(), Watches() or WatchesRawSource() to set them up") + } for _, w := range blder.watchesInput { + projected, err := blder.project(w.obj, w.objectProjection) + if err != nil { + return fmt.Errorf("failed to project for %T: %w", w.obj, err) + } allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) allPredicates = append(allPredicates, w.predicates...) - - // If the source of this watch is of type *source.Kind, project it. - if srckind, ok := w.src.(*source.Kind); ok { - typeForSrc, err := blder.project(srckind.Type, w.objectProjection) - if err != nil { - return err - } - srckind.Type = typeForSrc + if err := blder.ctrl.Watch(source.TypedKind(blder.mgr.GetCache(), projected, w.handler, allPredicates...)); err != nil { + return err } - - if err := blder.ctrl.Watch(w.src, w.eventhandler, allPredicates...); err != nil { + } + for _, src := range blder.rawSources { + if err := blder.ctrl.Watch(src); err != nil { return err } } return nil } -func (blder *Builder) getControllerName(gvk schema.GroupVersionKind) string { +func (blder *TypedBuilder[request]) getControllerName(gvk schema.GroupVersionKind, hasGVK bool) (string, error) { if blder.name != "" { - return blder.name + return blder.name, nil + } + if !hasGVK { + return "", errors.New("one of For() or Named() must be called") } - return strings.ToLower(gvk.Kind) + return strings.ToLower(gvk.Kind), nil } -func (blder *Builder) doController(r reconcile.Reconciler) error { +func (blder *TypedBuilder[request]) doController(r reconcile.TypedReconciler[request]) error { globalOpts := blder.mgr.GetControllerOptions() ctrlOptions := blder.ctrlOptions + if ctrlOptions.Reconciler != nil && r != nil { + return errors.New("reconciler was set via WithOptions() and via Build() or Complete()") + } if ctrlOptions.Reconciler == nil { ctrlOptions.Reconciler = r } // Retrieve the GVK from the object we're reconciling - // to prepopulate logger information, and to optionally generate a default name. - gvk, err := getGvk(blder.forInput.object, blder.mgr.GetScheme()) - if err != nil { - return err + // to pre-populate logger information, and to optionally generate a default name. + var gvk schema.GroupVersionKind + hasGVK := blder.forInput.object != nil + if hasGVK { + var err error + gvk, err = apiutil.GVKForObject(blder.forInput.object, blder.mgr.GetScheme()) + if err != nil { + return err + } } // Setup concurrency. - if ctrlOptions.MaxConcurrentReconciles == 0 { + if ctrlOptions.MaxConcurrentReconciles == 0 && hasGVK { groupKind := gvk.GroupKind().String() if concurrency, ok := globalOpts.GroupKindConcurrency[groupKind]; ok && concurrency > 0 { @@ -300,17 +420,47 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { } // Setup cache sync timeout. - if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout != nil { - ctrlOptions.CacheSyncTimeout = *globalOpts.CacheSyncTimeout + if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout > 0 { + ctrlOptions.CacheSyncTimeout = globalOpts.CacheSyncTimeout + } + + controllerName, err := blder.getControllerName(gvk, hasGVK) + if err != nil { + return err } // Setup the logger. - if ctrlOptions.Log == nil { - ctrlOptions.Log = blder.mgr.GetLogger() + if ctrlOptions.LogConstructor == nil { + log := blder.mgr.GetLogger().WithValues( + "controller", controllerName, + ) + if hasGVK { + log = log.WithValues( + "controllerGroup", gvk.Group, + "controllerKind", gvk.Kind, + ) + } + + ctrlOptions.LogConstructor = func(in *request) logr.Logger { + log := log + + if req, ok := any(in).(*reconcile.Request); ok && req != nil { + if hasGVK { + log = log.WithValues(gvk.Kind, klog.KRef(req.Namespace, req.Name)) + } + log = log.WithValues( + "namespace", req.Namespace, "name", req.Name, + ) + } + return log + } + } + + if blder.newController == nil { + blder.newController = controller.NewTyped[request] } - ctrlOptions.Log = ctrlOptions.Log.WithValues("reconciler group", gvk.Group, "reconciler kind", gvk.Kind) // Build the controller and return. - blder.ctrl, err = newController(blder.getControllerName(gvk), blder.mgr, ctrlOptions) + blder.ctrl, err = blder.newController(controllerName, blder.mgr, ctrlOptions) return err } diff --git a/pkg/builder/controller_test.go b/pkg/builder/controller_test.go index 62c5044e99..46e937d590 100644 --- a/pkg/builder/controller_test.go +++ b/pkg/builder/controller_test.go @@ -23,7 +23,7 @@ import ( "sync/atomic" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -33,10 +33,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -47,31 +48,39 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) -type typedNoop struct{} - -func (typedNoop) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, nil -} +var _ untypedWatchesInput = (*WatchesInput[struct{}])(nil) type testLogger struct { logr.Logger } -func (l *testLogger) WithName(_ string) logr.Logger { +func (l *testLogger) Init(logr.RuntimeInfo) { +} + +func (l *testLogger) Enabled(int) bool { + return true +} + +func (l *testLogger) Info(level int, msg string, keysAndValues ...interface{}) { +} + +func (l *testLogger) WithValues(keysAndValues ...interface{}) logr.LogSink { return l } -func (l *testLogger) WithValues(_ ...interface{}) logr.Logger { + +func (l *testLogger) WithName(name string) logr.LogSink { return l } -var _ = Describe("application", func() { - BeforeEach(func() { - newController = controller.New - }) +type empty struct{} +var _ = Describe("application", func() { noop := reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { return reconcile.Result{}, nil }) + typedNoop := reconcile.TypedFunc[empty](func(context.Context, empty) (reconcile.Result, error) { + return reconcile.Result{}, nil + }) Describe("New", func() { It("should return success if given valid objects", func() { @@ -101,18 +110,57 @@ var _ = Describe("application", func() { Expect(instance).To(BeNil()) }) - It("should return an error if For function is not called", func() { + It("should return an error if For and Named function are not called", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := ControllerManagedBy(m). + Watches(&appsv1.ReplicaSet{}, &handler.EnqueueRequestForObject{}). + Build(noop) + Expect(err).To(MatchError(ContainSubstring("one of For() or Named() must be called"))) + Expect(instance).To(BeNil()) + }) + + It("should return an error when using Owns without For", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) instance, err := ControllerManagedBy(m). + Named("my_controller"). Owns(&appsv1.ReplicaSet{}). Build(noop) - Expect(err).To(MatchError(ContainSubstring("must provide an object for reconciliation"))) + Expect(err).To(MatchError(ContainSubstring("Owns() can only be used together with For()"))) + Expect(instance).To(BeNil()) + + }) + + It("should return an error when there are no watches", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := ControllerManagedBy(m). + Named("my_new_controller"). + Build(noop) + Expect(err).To(MatchError(ContainSubstring("there are no watches configured, controller will never get triggered. Use For(), Owns(), Watches() or WatchesRawSource() to set them up"))) Expect(instance).To(BeNil()) }) + It("should allow creating a controller without calling For", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := ControllerManagedBy(m). + Named("my_other_controller"). + Watches(&appsv1.ReplicaSet{}, &handler.EnqueueRequestForObject{}). + Build(noop) + Expect(err).NotTo(HaveOccurred()) + Expect(instance).NotTo(BeNil()) + }) + It("should return an error if there is no GVK for an object, and thus we can't default the controller name", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) @@ -131,20 +179,70 @@ var _ = Describe("application", func() { // manifest when we try to default the controller name, which is good to double check. }) + It("should return error if in For is used with a custom request type", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := TypedControllerManagedBy[empty](m). + For(&appsv1.ReplicaSet{}). + Named("last_controller"). + Build(typedNoop) + Expect(err).To(MatchError(ContainSubstring("For() can only be used with reconcile.Request, got builder.empty"))) + Expect(instance).To(BeNil()) + }) + + It("should return error if in Owns is used with a custom request type", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := TypedControllerManagedBy[empty](m). + Named("my_controller-0"). + Owns(&appsv1.ReplicaSet{}). + Build(typedNoop) + // If we ever allow Owns() without For() we need to update the code to error + // out on Owns() if the request type is different from reconcile.Request just + // like we do in For(). + Expect(err).To(MatchError("Owns() can only be used together with For()")) + Expect(instance).To(BeNil()) + }) + + It("should build a controller with a custom request type", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := TypedControllerManagedBy[empty](m). + Named("my_controller-1"). + WatchesRawSource( + source.TypedKind( + m.GetCache(), + &appsv1.ReplicaSet{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, rs *appsv1.ReplicaSet) []empty { + return []empty{{}} + }), + ), + ). + Build(typedNoop) + Expect(err).NotTo(HaveOccurred()) + Expect(instance).NotTo(BeNil()) + }) + It("should return an error if it cannot create the controller", func() { - newController = func(name string, mgr manager.Manager, options controller.Options) ( - controller.Controller, error) { - return nil, fmt.Errorf("expected error") - } By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - instance, err := ControllerManagedBy(m). + builder := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). - Owns(&appsv1.ReplicaSet{}). - Build(noop) + Owns(&appsv1.ReplicaSet{}) + builder.newController = func(name string, mgr manager.Manager, options controller.Options) ( + controller.Controller, error) { + return nil, fmt.Errorf("expected error") + } + instance, err := builder.Build(noop) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("expected error")) Expect(instance).To(BeNil()) @@ -152,7 +250,7 @@ var _ = Describe("application", func() { It("should override max concurrent reconcilers during creation of controller", func() { const maxConcurrentReconciles = 5 - newController = func(name string, mgr manager.Manager, options controller.Options) ( + newController := func(name string, mgr manager.Manager, options controller.Options) ( controller.Controller, error) { if options.MaxConcurrentReconciles == maxConcurrentReconciles { return controller.New(name, mgr, options) @@ -164,18 +262,21 @@ var _ = Describe("application", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - instance, err := ControllerManagedBy(m). + builder := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). + Named("replicaset-4"). Owns(&appsv1.ReplicaSet{}). - WithOptions(controller.Options{MaxConcurrentReconciles: maxConcurrentReconciles}). - Build(noop) + WithOptions(controller.Options{MaxConcurrentReconciles: maxConcurrentReconciles}) + builder.newController = newController + + instance, err := builder.Build(noop) Expect(err).NotTo(HaveOccurred()) Expect(instance).NotTo(BeNil()) }) It("should override max concurrent reconcilers during creation of controller, when using", func() { const maxConcurrentReconciles = 10 - newController = func(name string, mgr manager.Manager, options controller.Options) ( + newController := func(name string, mgr manager.Manager, options controller.Options) ( controller.Controller, error) { if options.MaxConcurrentReconciles == maxConcurrentReconciles { return controller.New(name, mgr, options) @@ -185,7 +286,7 @@ var _ = Describe("application", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{ - Controller: v1alpha1.ControllerConfigurationSpec{ + Controller: config.Controller{ GroupKindConcurrency: map[string]int{ "ReplicaSet.apps": maxConcurrentReconciles, }, @@ -193,17 +294,20 @@ var _ = Describe("application", func() { }) Expect(err).NotTo(HaveOccurred()) - instance, err := ControllerManagedBy(m). + builder := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). - Owns(&appsv1.ReplicaSet{}). - Build(noop) + Named("replicaset-3"). + Owns(&appsv1.ReplicaSet{}) + builder.newController = newController + + instance, err := builder.Build(noop) Expect(err).NotTo(HaveOccurred()) Expect(instance).NotTo(BeNil()) }) It("should override rate limiter during creation of controller", func() { - rateLimiter := workqueue.DefaultItemBasedRateLimiter() - newController = func(name string, mgr manager.Manager, options controller.Options) (controller.Controller, error) { + rateLimiter := workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]() + newController := func(name string, mgr manager.Manager, options controller.Options) (controller.Controller, error) { if options.RateLimiter == rateLimiter { return controller.New(name, mgr, options) } @@ -214,57 +318,57 @@ var _ = Describe("application", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - instance, err := ControllerManagedBy(m). + builder := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). + Named("replicaset-2"). Owns(&appsv1.ReplicaSet{}). - WithOptions(controller.Options{RateLimiter: rateLimiter}). - Build(noop) + WithOptions(controller.Options{RateLimiter: rateLimiter}) + builder.newController = newController + + instance, err := builder.Build(noop) Expect(err).NotTo(HaveOccurred()) Expect(instance).NotTo(BeNil()) }) It("should override logger during creation of controller", func() { - logger := &testLogger{} - newController = func(name string, mgr manager.Manager, options controller.Options) (controller.Controller, error) { - if options.Log == logger { + newController := func(name string, mgr manager.Manager, options controller.Options) (controller.Controller, error) { + if options.LogConstructor(nil).GetSink() == logger { return controller.New(name, mgr, options) } - return nil, fmt.Errorf("logger expected %T but found %T", logger, options.Log) + return nil, fmt.Errorf("logger expected %T but found %T", logger, options.LogConstructor) } By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - instance, err := ControllerManagedBy(m). + builder := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). + Named("replicaset-0"). Owns(&appsv1.ReplicaSet{}). - WithLogger(logger). - Build(noop) + WithLogConstructor(func(request *reconcile.Request) logr.Logger { + return logr.New(logger) + }) + builder.newController = newController + instance, err := builder.Build(noop) Expect(err).NotTo(HaveOccurred()) Expect(instance).NotTo(BeNil()) }) - It("should prefer reconciler from options during creation of controller", func() { - newController = func(name string, mgr manager.Manager, options controller.Options) (controller.Controller, error) { - if options.Reconciler != (typedNoop{}) { - return nil, fmt.Errorf("Custom reconciler expected %T but found %T", typedNoop{}, options.Reconciler) - } - return controller.New(name, mgr, options) - } - + It("should not allow multiple reconcilers during creation of controller", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - instance, err := ControllerManagedBy(m). + builder := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). + Named("replicaset-1"). Owns(&appsv1.ReplicaSet{}). - WithOptions(controller.Options{Reconciler: typedNoop{}}). - Build(noop) - Expect(err).NotTo(HaveOccurred()) - Expect(instance).NotTo(BeNil()) + WithOptions(controller.Options{Reconciler: noop}) + instance, err := builder.Build(noop) + Expect(err).To(HaveOccurred()) + Expect(instance).To(BeNil()) }) It("should allow multiple controllers for the same kind", func() { @@ -289,6 +393,7 @@ var _ = Describe("application", func() { By("creating the 2nd controller") ctrl2, err := ControllerManagedBy(m). For(&TestDefaultValidator{}). + Named("test-default-validator-1"). Owns(&appsv1.ReplicaSet{}). Build(noop) Expect(err).NotTo(HaveOccurred()) @@ -297,37 +402,64 @@ var _ = Describe("application", func() { }) Describe("Start with ControllerManagedBy", func() { - It("should Reconcile Owns objects", func() { + It("should Reconcile Owns objects", func(ctx SpecContext) { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) bldr := ControllerManagedBy(m). For(&appsv1.Deployment{}). + Named("deployment-0"). Owns(&appsv1.ReplicaSet{}) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() doReconcileTest(ctx, "3", m, false, bldr) - }, 10) + }) - It("should Reconcile Watches objects", func() { + It("should Reconcile Owns objects for every owner", func(ctx SpecContext) { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + bldr := ControllerManagedBy(m). + For(&appsv1.Deployment{}). + Named("deployment-1"). + Owns(&appsv1.ReplicaSet{}, MatchEveryOwner) + + doReconcileTest(ctx, "12", m, false, bldr) + }) + + It("should Reconcile Watches objects", func(ctx SpecContext) { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) bldr := ControllerManagedBy(m). For(&appsv1.Deployment{}). Watches( // Equivalent of Owns - &source.Kind{Type: &appsv1.ReplicaSet{}}, - &handler.EnqueueRequestForOwner{OwnerType: &appsv1.Deployment{}, IsController: true}) + &appsv1.ReplicaSet{}, + handler.EnqueueRequestForOwner(m.GetScheme(), m.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), + ) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() doReconcileTest(ctx, "4", m, true, bldr) - }, 10) + }) + + It("should Reconcile without For", func(ctx SpecContext) { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + bldr := ControllerManagedBy(m). + Named("Deployment"). + Named("deployment-2"). + Watches( // Equivalent of For + &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}). + Watches( // Equivalent of Owns + &appsv1.ReplicaSet{}, + handler.EnqueueRequestForOwner(m.GetScheme(), m.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), + ) + + doReconcileTest(ctx, "9", m, true, bldr) + }) }) Describe("Set custom predicates", func() { - It("should execute registered predicates only for assigned kind", func() { + It("should execute registered predicates only for assigned kind", func(ctx SpecContext) { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -373,11 +505,10 @@ var _ = Describe("application", func() { bldr := ControllerManagedBy(m). For(&appsv1.Deployment{}, WithPredicates(deployPrct)). + Named("deployment-3"). Owns(&appsv1.ReplicaSet{}, WithPredicates(replicaSetPrct)). WithEventFilter(allPrct) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() doReconcileTest(ctx, "5", m, true, bldr) Expect(deployPrctExecuted).To(BeTrue(), "Deploy predicated should be called at least once") @@ -396,24 +527,22 @@ var _ = Describe("application", func() { Expect(err).NotTo(HaveOccurred()) }) - It("should support multiple controllers watching the same metadata kind", func() { - bldr1 := ControllerManagedBy(mgr).For(&appsv1.Deployment{}, OnlyMetadata) - bldr2 := ControllerManagedBy(mgr).For(&appsv1.Deployment{}, OnlyMetadata) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should support multiple controllers watching the same metadata kind", func(ctx SpecContext) { + bldr1 := ControllerManagedBy(mgr).For(&appsv1.Deployment{}, OnlyMetadata).Named("deployment-4") + bldr2 := ControllerManagedBy(mgr).For(&appsv1.Deployment{}, OnlyMetadata).Named("deployment-5") doReconcileTest(ctx, "6", mgr, true, bldr1, bldr2) }) - It("should support watching For, Owns, and Watch as metadata", func() { + It("should support watching For, Owns, and Watch as metadata", func(ctx SpecContext) { statefulSetMaps := make(chan *metav1.PartialObjectMetadata) bldr := ControllerManagedBy(mgr). For(&appsv1.Deployment{}, OnlyMetadata). + Named("deployment-6"). Owns(&appsv1.ReplicaSet{}, OnlyMetadata). - Watches(&source.Kind{Type: &appsv1.StatefulSet{}}, - handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { + Watches(&appsv1.StatefulSet{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { defer GinkgoRecover() ometa := o.(*metav1.PartialObjectMetadata) @@ -429,8 +558,6 @@ var _ = Describe("application", func() { }), OnlyMetadata) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() doReconcileTest(ctx, "8", mgr, true, bldr) By("Creating a new stateful set") @@ -459,7 +586,7 @@ var _ = Describe("application", func() { }, }, } - err := mgr.GetClient().Create(context.TODO(), set) + err := mgr.GetClient().Create(ctx, set) Expect(err).NotTo(HaveOccurred()) By("Checking that the mapping function has been called") @@ -492,21 +619,21 @@ type nonTypedOnlyCache struct { cache.Cache } -func (c *nonTypedOnlyCache) GetInformer(ctx context.Context, obj client.Object) (cache.Informer, error) { +func (c *nonTypedOnlyCache) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { switch obj.(type) { case (*metav1.PartialObjectMetadata): - return c.Cache.GetInformer(ctx, obj) + return c.Cache.GetInformer(ctx, obj, opts...) default: return nil, fmt.Errorf("did not want to provide an informer for normal type %T", obj) } } -func (c *nonTypedOnlyCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (cache.Informer, error) { +func (c *nonTypedOnlyCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...cache.InformerGetOption) (cache.Informer, error) { return nil, fmt.Errorf("don't try to sidestep the restriction on informer types by calling GetInformerForKind") } // TODO(directxman12): this function has too many arguments, and the whole // "nameSuffix" think is a bit of a hack It should be cleaned up significantly by someone with a bit of time. -func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager, complete bool, blders ...*Builder) { +func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager, complete bool, blders ...*TypedBuilder[reconcile.Request]) { deployName := "deploy-name-" + nameSuffix rsName := "rs-name-" + nameSuffix @@ -539,7 +666,6 @@ func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager go func() { defer GinkgoRecover() Expect(mgr.Start(ctx)).NotTo(HaveOccurred()) - By("Stopping the application") }() By("Creating a Deployment") @@ -566,7 +692,7 @@ func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager }, }, } - err := mgr.GetClient().Create(context.TODO(), dep) + err := mgr.GetClient().Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("Waiting for the Deployment Reconcile") @@ -575,7 +701,6 @@ func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager By("Creating a ReplicaSet") // Expect a Reconcile when an Owned object is managedObjects. - t := true rs := &appsv1.ReplicaSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -586,7 +711,7 @@ func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager Name: deployName, Kind: "Deployment", APIVersion: "apps/v1", - Controller: &t, + Controller: ptr.To(true), UID: dep.UID, }, }, @@ -596,7 +721,7 @@ func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager Template: dep.Spec.Template, }, } - err = mgr.GetClient().Create(context.TODO(), rs) + err = mgr.GetClient().Create(ctx, rs) Expect(err).NotTo(HaveOccurred()) By("Waiting for the ReplicaSet Reconcile") diff --git a/pkg/builder/example_test.go b/pkg/builder/example_test.go index 955c46b562..e265853987 100644 --- a/pkg/builder/example_test.go +++ b/pkg/builder/example_test.go @@ -38,7 +38,7 @@ import ( func ExampleBuilder_metadata_only() { logf.SetLogger(zap.New()) - var log = logf.Log.WithName("builder-examples") + log := logf.Log.WithName("builder-examples") mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{}) if err != nil { @@ -95,7 +95,7 @@ func ExampleBuilder_metadata_only() { func ExampleBuilder() { logf.SetLogger(zap.New()) - var log = logf.Log.WithName("builder-examples") + log := logf.Log.WithName("builder-examples") mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{}) if err != nil { @@ -107,7 +107,9 @@ func ExampleBuilder() { ControllerManagedBy(mgr). // Create the ControllerManagedBy For(&appsv1.ReplicaSet{}). // ReplicaSet is the Application API Owns(&corev1.Pod{}). // ReplicaSet owns Pods created by it - Complete(&ReplicaSetReconciler{}) + Complete(&ReplicaSetReconciler{ + Client: mgr.GetClient(), + }) if err != nil { log.Error(err, "could not create controller") os.Exit(1) @@ -155,8 +157,3 @@ func (a *ReplicaSetReconciler) Reconcile(ctx context.Context, req reconcile.Requ return reconcile.Result{}, nil } - -func (a *ReplicaSetReconciler) InjectClient(c client.Client) error { - a.Client = c - return nil -} diff --git a/pkg/builder/example_webhook_test.go b/pkg/builder/example_webhook_test.go index 63333a2478..c26eba8a13 100644 --- a/pkg/builder/example_webhook_test.go +++ b/pkg/builder/example_webhook_test.go @@ -24,16 +24,10 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" examplegroup "sigs.k8s.io/controller-runtime/examples/crd/pkg" ) -// examplegroup.ChaosPod has implemented both admission.Defaulter and -// admission.Validator interfaces. -var _ admission.Defaulter = &examplegroup.ChaosPod{} -var _ admission.Validator = &examplegroup.ChaosPod{} - // This example use webhook builder to create a simple webhook that is managed // by a manager for CRD ChaosPod. And then start the manager. func ExampleWebhookBuilder() { diff --git a/pkg/builder/options.go b/pkg/builder/options.go index 7bb4273094..b907b5d020 100644 --- a/pkg/builder/options.go +++ b/pkg/builder/options.go @@ -28,7 +28,7 @@ type ForOption interface { ApplyToFor(*ForInput) } -// OwnsOption is some configuration that modifies options for a owns request. +// OwnsOption is some configuration that modifies options for an owns request. type OwnsOption interface { // ApplyToOwns applies this configuration to the given owns input. ApplyToOwns(*OwnsInput) @@ -37,7 +37,7 @@ type OwnsOption interface { // WatchesOption is some configuration that modifies options for a watches request. type WatchesOption interface { // ApplyToWatches applies this configuration to the given watches options. - ApplyToWatches(*WatchesInput) + ApplyToWatches(untypedWatchesInput) } // }}} @@ -67,8 +67,8 @@ func (w Predicates) ApplyToOwns(opts *OwnsInput) { } // ApplyToWatches applies this configuration to the given WatchesInput options. -func (w Predicates) ApplyToWatches(opts *WatchesInput) { - opts.predicates = w.predicates +func (w Predicates) ApplyToWatches(opts untypedWatchesInput) { + opts.setPredicates(w.predicates) } var _ ForOption = &Predicates{} @@ -79,8 +79,8 @@ var _ WatchesOption = &Predicates{} // {{{ For & Owns Dual-Type options -// asProjection configures the projection (currently only metadata) on the input. -// Currently only metadata is supported. We might want to expand +// projectAs configures the projection on the input. +// Currently only OnlyMetadata is supported. We might want to expand // this to arbitrary non-special local projections in the future. type projectAs objectProjection @@ -95,18 +95,41 @@ func (p projectAs) ApplyToOwns(opts *OwnsInput) { } // ApplyToWatches applies this configuration to the given WatchesInput options. -func (p projectAs) ApplyToWatches(opts *WatchesInput) { - opts.objectProjection = objectProjection(p) +func (p projectAs) ApplyToWatches(opts untypedWatchesInput) { + opts.setObjectProjection(objectProjection(p)) } var ( // OnlyMetadata tells the controller to *only* cache metadata, and to watch - // the the API server in metadata-only form. This is useful when watching + // the API server in metadata-only form. This is useful when watching // lots of objects, really big objects, or objects for which you only know - // the the GVK, but not the structure. You'll need to pass + // the GVK, but not the structure. You'll need to pass // metav1.PartialObjectMetadata to the client when fetching objects in your // reconciler, otherwise you'll end up with a duplicate structured or // unstructured cache. + // + // When watching a resource with OnlyMetadata, for example the v1.Pod, you + // should not Get and List using the v1.Pod type. Instead, you should use + // the special metav1.PartialObjectMetadata type. + // + // ❌ Incorrect: + // + // pod := &v1.Pod{} + // mgr.GetClient().Get(ctx, nsAndName, pod) + // + // ✅ Correct: + // + // pod := &metav1.PartialObjectMetadata{} + // pod.SetGroupVersionKind(schema.GroupVersionKind{ + // Group: "", + // Version: "v1", + // Kind: "Pod", + // }) + // mgr.GetClient().Get(ctx, nsAndName, pod) + // + // In the first case, controller-runtime will create another cache for the + // concrete type on top of the metadata cache; this increases memory + // consumption and leads to race conditions as caches are not in sync. OnlyMetadata = projectAs(projectAsMetadata) _ ForOption = OnlyMetadata @@ -115,3 +138,19 @@ var ( ) // }}} + +// MatchEveryOwner determines whether the watch should be filtered based on +// controller ownership. As in, when the OwnerReference.Controller field is set. +// +// If passed as an option, +// the handler receives notification for every owner of the object with the given type. +// If unset (default), the handler receives notification only for the first +// OwnerReference with `Controller: true`. +var MatchEveryOwner = &matchEveryOwner{} + +type matchEveryOwner struct{} + +// ApplyToOwns applies this configuration to the given OwnsInput options. +func (o matchEveryOwner) ApplyToOwns(opts *OwnsInput) { + opts.matchEveryOwner = true +} diff --git a/pkg/builder/webhook.go b/pkg/builder/webhook.go index dcfa787bb2..6f4726d274 100644 --- a/pkg/builder/webhook.go +++ b/pkg/builder/webhook.go @@ -17,14 +17,19 @@ limitations under the License. package builder import ( + "context" "errors" "net/http" "net/url" + "regexp" "strings" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -33,15 +38,23 @@ import ( // WebhookBuilder builds a Webhook. type WebhookBuilder struct { - apiType runtime.Object - withDefaulter admission.CustomDefaulter - withValidator admission.CustomValidator - gvk schema.GroupVersionKind - mgr manager.Manager - config *rest.Config + apiType runtime.Object + customDefaulter admission.CustomDefaulter + customDefaulterOpts []admission.DefaulterOption + customValidator admission.CustomValidator + customPath string + customValidatorCustomPath string + customDefaulterCustomPath string + gvk schema.GroupVersionKind + mgr manager.Manager + config *rest.Config + recoverPanic *bool + logConstructor func(base logr.Logger, req *admission.Request) logr.Logger + contextFunc func(context.Context, *http.Request) context.Context + err error } -// WebhookManagedBy allows inform its manager.Manager. +// WebhookManagedBy returns a new webhook builder. func WebhookManagedBy(m manager.Manager) *WebhookBuilder { return &WebhookBuilder{mgr: m} } @@ -52,19 +65,64 @@ func WebhookManagedBy(m manager.Manager) *WebhookBuilder { // If the given object implements the admission.Defaulter interface, a MutatingWebhook will be wired for this type. // If the given object implements the admission.Validator interface, a ValidatingWebhook will be wired for this type. func (blder *WebhookBuilder) For(apiType runtime.Object) *WebhookBuilder { + if blder.apiType != nil { + blder.err = errors.New("For(...) should only be called once, could not assign multiple objects for webhook registration") + } blder.apiType = apiType return blder } -// WithDefaulter takes a admission.WithDefaulter interface, a MutatingWebhook will be wired for this type. -func (blder *WebhookBuilder) WithDefaulter(defaulter admission.CustomDefaulter) *WebhookBuilder { - blder.withDefaulter = defaulter +// WithDefaulter takes an admission.CustomDefaulter interface, a MutatingWebhook with the provided opts (admission.DefaulterOption) +// will be wired for this type. +func (blder *WebhookBuilder) WithDefaulter(defaulter admission.CustomDefaulter, opts ...admission.DefaulterOption) *WebhookBuilder { + blder.customDefaulter = defaulter + blder.customDefaulterOpts = opts return blder } -// WithValidator takes a admission.WithValidator interface, a ValidatingWebhook will be wired for this type. +// WithValidator takes a admission.CustomValidator interface, a ValidatingWebhook will be wired for this type. func (blder *WebhookBuilder) WithValidator(validator admission.CustomValidator) *WebhookBuilder { - blder.withValidator = validator + blder.customValidator = validator + return blder +} + +// WithLogConstructor overrides the webhook's LogConstructor. +func (blder *WebhookBuilder) WithLogConstructor(logConstructor func(base logr.Logger, req *admission.Request) logr.Logger) *WebhookBuilder { + blder.logConstructor = logConstructor + return blder +} + +// WithContextFunc overrides the webhook's WithContextFunc. +func (blder *WebhookBuilder) WithContextFunc(contextFunc func(context.Context, *http.Request) context.Context) *WebhookBuilder { + blder.contextFunc = contextFunc + return blder +} + +// RecoverPanic indicates whether panics caused by the webhook should be recovered. +// Defaults to true. +func (blder *WebhookBuilder) RecoverPanic(recoverPanic bool) *WebhookBuilder { + blder.recoverPanic = &recoverPanic + return blder +} + +// WithCustomPath overrides the webhook's default path by the customPath +// +// Deprecated: WithCustomPath should not be used anymore. +// Please use WithValidatorCustomPath or WithDefaulterCustomPath instead. +func (blder *WebhookBuilder) WithCustomPath(customPath string) *WebhookBuilder { + blder.customPath = customPath + return blder +} + +// WithValidatorCustomPath overrides the path of the Validator. +func (blder *WebhookBuilder) WithValidatorCustomPath(customPath string) *WebhookBuilder { + blder.customValidatorCustomPath = customPath + return blder +} + +// WithDefaulterCustomPath overrides the path of the Defaulter. +func (blder *WebhookBuilder) WithDefaulterCustomPath(customPath string) *WebhookBuilder { + blder.customDefaulterCustomPath = customPath return blder } @@ -73,6 +131,9 @@ func (blder *WebhookBuilder) Complete() error { // Set the Config blder.loadRestConfig() + // Configure the default LogConstructor + blder.setLogConstructor() + // Set the Webhook if needed return blder.registerWebhooks() } @@ -83,33 +144,84 @@ func (blder *WebhookBuilder) loadRestConfig() { } } +func (blder *WebhookBuilder) setLogConstructor() { + if blder.logConstructor == nil { + blder.logConstructor = func(base logr.Logger, req *admission.Request) logr.Logger { + log := base.WithValues( + "webhookGroup", blder.gvk.Group, + "webhookKind", blder.gvk.Kind, + ) + if req != nil { + return log.WithValues( + blder.gvk.Kind, klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + "resource", req.Resource, "user", req.UserInfo.Username, + "requestID", req.UID, + ) + } + return log + } + } +} + +func (blder *WebhookBuilder) isThereCustomPathConflict() bool { + return (blder.customPath != "" && blder.customDefaulter != nil && blder.customValidator != nil) || (blder.customPath != "" && blder.customDefaulterCustomPath != "") || (blder.customPath != "" && blder.customValidatorCustomPath != "") +} + func (blder *WebhookBuilder) registerWebhooks() error { typ, err := blder.getType() if err != nil { return err } - // Create webhook(s) for each type blder.gvk, err = apiutil.GVKForObject(typ, blder.mgr.GetScheme()) if err != nil { return err } - blder.registerDefaultingWebhook() - blder.registerValidatingWebhook() + if blder.isThereCustomPathConflict() { + return errors.New("only one of CustomDefaulter or CustomValidator should be set when using WithCustomPath. Otherwise, WithDefaulterCustomPath() and WithValidatorCustomPath() should be used") + } + if blder.customPath != "" { + // isThereCustomPathConflict() already checks for potential conflicts. + // Since we are sure that only one of customDefaulter or customValidator will be used, + // we can set both customDefaulterCustomPath and validatingCustomPath. + blder.customDefaulterCustomPath = blder.customPath + blder.customValidatorCustomPath = blder.customPath + } + + // Register webhook(s) for type + err = blder.registerDefaultingWebhook() + if err != nil { + return err + } + + err = blder.registerValidatingWebhook() + if err != nil { + return err + } err = blder.registerConversionWebhook() if err != nil { return err } - return nil + return blder.err } -// registerDefaultingWebhook registers a defaulting webhook if th. -func (blder *WebhookBuilder) registerDefaultingWebhook() { +// registerDefaultingWebhook registers a defaulting webhook if necessary. +func (blder *WebhookBuilder) registerDefaultingWebhook() error { mwh := blder.getDefaultingWebhook() if mwh != nil { + mwh.LogConstructor = blder.logConstructor + mwh.WithContextFunc = blder.contextFunc path := generateMutatePath(blder.gvk) + if blder.customDefaulterCustomPath != "" { + generatedCustomPath, err := generateCustomPath(blder.customDefaulterCustomPath) + if err != nil { + return err + } + path = generatedCustomPath + } // Checking if the path is already registered. // If so, just skip it. @@ -120,25 +232,35 @@ func (blder *WebhookBuilder) registerDefaultingWebhook() { blder.mgr.GetWebhookServer().Register(path, mwh) } } + + return nil } func (blder *WebhookBuilder) getDefaultingWebhook() *admission.Webhook { - if defaulter := blder.withDefaulter; defaulter != nil { - return admission.WithCustomDefaulter(blder.apiType, defaulter) - } - if defaulter, ok := blder.apiType.(admission.Defaulter); ok { - return admission.DefaultingWebhookFor(defaulter) + if defaulter := blder.customDefaulter; defaulter != nil { + w := admission.WithCustomDefaulter(blder.mgr.GetScheme(), blder.apiType, defaulter, blder.customDefaulterOpts...) + if blder.recoverPanic != nil { + w = w.WithRecoverPanic(*blder.recoverPanic) + } + return w } - log.Info( - "skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called", - "GVK", blder.gvk) return nil } -func (blder *WebhookBuilder) registerValidatingWebhook() { +// registerValidatingWebhook registers a validating webhook if necessary. +func (blder *WebhookBuilder) registerValidatingWebhook() error { vwh := blder.getValidatingWebhook() if vwh != nil { + vwh.LogConstructor = blder.logConstructor + vwh.WithContextFunc = blder.contextFunc path := generateValidatePath(blder.gvk) + if blder.customValidatorCustomPath != "" { + generatedCustomPath, err := generateCustomPath(blder.customValidatorCustomPath) + if err != nil { + return err + } + path = generatedCustomPath + } // Checking if the path is already registered. // If so, just skip it. @@ -149,32 +271,32 @@ func (blder *WebhookBuilder) registerValidatingWebhook() { blder.mgr.GetWebhookServer().Register(path, vwh) } } + + return nil } func (blder *WebhookBuilder) getValidatingWebhook() *admission.Webhook { - if validator := blder.withValidator; validator != nil { - return admission.WithCustomValidator(blder.apiType, validator) - } - if validator, ok := blder.apiType.(admission.Validator); ok { - return admission.ValidatingWebhookFor(validator) + if validator := blder.customValidator; validator != nil { + w := admission.WithCustomValidator(blder.mgr.GetScheme(), blder.apiType, validator) + if blder.recoverPanic != nil { + w = w.WithRecoverPanic(*blder.recoverPanic) + } + return w } - log.Info( - "skip registering a validating webhook, object does not implement admission.Validator or WithValidator wasn't called", - "GVK", blder.gvk) return nil } func (blder *WebhookBuilder) registerConversionWebhook() error { ok, err := conversion.IsConvertible(blder.mgr.GetScheme(), blder.apiType) if err != nil { - log.Error(err, "conversion check failed", "object", blder.apiType) + log.Error(err, "conversion check failed", "GVK", blder.gvk) return err } if ok { if !blder.isAlreadyHandled("/convert") { - blder.mgr.GetWebhookServer().Register("/convert", &conversion.Webhook{}) + blder.mgr.GetWebhookServer().Register("/convert", conversion.NewWebhookHandler(blder.mgr.GetScheme())) } - log.Info("conversion webhook enabled", "object", blder.apiType) + log.Info("Conversion webhook enabled", "GVK", blder.gvk) } return nil @@ -188,10 +310,10 @@ func (blder *WebhookBuilder) getType() (runtime.Object, error) { } func (blder *WebhookBuilder) isAlreadyHandled(path string) bool { - if blder.mgr.GetWebhookServer().WebhookMux == nil { + if blder.mgr.GetWebhookServer().WebhookMux() == nil { return false } - h, p := blder.mgr.GetWebhookServer().WebhookMux.Handler(&http.Request{URL: &url.URL{Path: path}}) + h, p := blder.mgr.GetWebhookServer().WebhookMux().Handler(&http.Request{URL: &url.URL{Path: path}}) if p == path && h != nil { return true } @@ -207,3 +329,14 @@ func generateValidatePath(gvk schema.GroupVersionKind) string { return "/validate-" + strings.ReplaceAll(gvk.Group, ".", "-") + "-" + gvk.Version + "-" + strings.ToLower(gvk.Kind) } + +const webhookPathStringValidation = `^((/[a-zA-Z0-9-_]+)+|/)$` + +var validWebhookPathRegex = regexp.MustCompile(webhookPathStringValidation) + +func generateCustomPath(customPath string) (string, error) { + if !validWebhookPathRegex.MatchString(customPath) { + return "", errors.New("customPath \"" + customPath + "\" does not match this regex: " + webhookPathStringValidation) + } + return customPath, nil +} diff --git a/pkg/builder/webhook_test.go b/pkg/builder/webhook_test.go index 80703a38ff..72538ef7bf 100644 --- a/pkg/builder/webhook_test.go +++ b/pkg/builder/webhook_test.go @@ -20,23 +20,43 @@ import ( "context" "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" "strings" - . "github.com/onsi/ginkgo" + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/controller" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/scheme" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) +const ( + admissionReviewGV = `{ + "kind":"AdmissionReview", + "apiVersion":"admission.k8s.io/` + + svcBaseAddr = "http://svc-name.svc-ns.svc" + + customPath = "/custom-path" + + userAgentHeader = "User-Agent" + userAgentCtxKey agentCtxKey = "UserAgent" + userAgentValue = "test" +) + +type agentCtxKey string + var _ = Describe("webhook", func() { Describe("New", func() { Context("v1 AdmissionReview", func() { @@ -49,18 +69,23 @@ var _ = Describe("webhook", func() { }) func runTests(admissionReviewVersion string) { - var stop chan struct{} + var ( + stop chan struct{} + logBuffer *gbytes.Buffer + testingLogger logr.Logger + ) BeforeEach(func() { stop = make(chan struct{}) - newController = controller.New + logBuffer = gbytes.NewBuffer() + testingLogger = zap.New(zap.JSONEncoder(), zap.WriteTo(io.MultiWriter(logBuffer, GinkgoWriter))) }) AfterEach(func() { close(stop) }) - It("should scaffold a defaulting webhook if the type implements the Defaulter interface", func() { + It("should scaffold a custom defaulting webhook", func(specCtx SpecContext) { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -73,27 +98,30 @@ func runTests(admissionReviewVersion string) { err = WebhookManagedBy(m). For(&TestDefaulter{}). + WithDefaulter(&TestCustomDefaulter{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). Complete() ExpectWithOffset(1, err).NotTo(HaveOccurred()) svr := m.GetWebhookServer() ExpectWithOffset(1, svr).NotTo(BeNil()) - reader := strings.NewReader(`{ - "kind":"AdmissionReview", - "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", "request":{ "uid":"07e52e8d-4513-11e9-a716-42010a800270", "kind":{ - "group":"", + "group":"foo.test.org", "version":"v1", "kind":"TestDefaulter" }, "resource":{ - "group":"", + "group":"foo.test.org", "version":"v1", "resource":"testdefaulter" }, "namespace":"default", + "name":"foo", "operation":"CREATE", "object":{ "replica":1 @@ -102,10 +130,8 @@ func runTests(admissionReviewVersion string) { } }`) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -113,28 +139,29 @@ func runTests(admissionReviewVersion string) { By("sending a request to a mutating webhook path") path := generateMutatePath(testDefaulterGVK) - req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w := httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) By("sanity checking the response contains reasonable fields") ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"patch":`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Defaulting object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testdefaulter"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) By("sending a request to a validating webhook path that doesn't exist") path = generateValidatePath(testDefaulterGVK) _, err = reader.Seek(0, 0) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w = httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) }) - It("should scaffold a defaulting webhook with a custom defaulter", func() { + It("should scaffold a custom defaulting webhook with a custom path", func(specCtx SpecContext) { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -145,30 +172,34 @@ func runTests(admissionReviewVersion string) { err = builder.AddToScheme(m.GetScheme()) ExpectWithOffset(1, err).NotTo(HaveOccurred()) + customPath := "/custom-defaulting-path" err = WebhookManagedBy(m). - WithDefaulter(&TestCustomDefaulter{}). For(&TestDefaulter{}). + WithDefaulter(&TestCustomDefaulter{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + WithDefaulterCustomPath(customPath). Complete() ExpectWithOffset(1, err).NotTo(HaveOccurred()) svr := m.GetWebhookServer() ExpectWithOffset(1, svr).NotTo(BeNil()) - reader := strings.NewReader(`{ - "kind":"AdmissionReview", - "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", "request":{ "uid":"07e52e8d-4513-11e9-a716-42010a800270", "kind":{ - "group":"", + "group":"foo.test.org", "version":"v1", "kind":"TestDefaulter" }, "resource":{ - "group":"", + "group":"foo.test.org", "version":"v1", "resource":"testdefaulter" }, "namespace":"default", + "name":"foo", "operation":"CREATE", "object":{ "replica":1 @@ -177,39 +208,103 @@ func runTests(admissionReviewVersion string) { } }`) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) } - By("sending a request to a mutating webhook path") - path := generateMutatePath(testDefaulterGVK) - req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + By("sending a request to a mutating webhook path that have been overriten by a custom path") + path, err := generateCustomPath(customPath) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w := httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) By("sanity checking the response contains reasonable fields") ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"patch":`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Defaulting object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testdefaulter"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) - By("sending a request to a validating webhook path that doesn't exist") - path = generateValidatePath(testDefaulterGVK) + By("sending a request to a mutating webhook path") + path = generateMutatePath(testDefaulterGVK) _, err = reader.Seek(0, 0) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w = httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) }) - It("should scaffold a validating webhook if the type implements the Validator interface", func() { + It("should scaffold a custom defaulting webhook which recovers from panics", func(specCtx SpecContext) { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testDefaulterGVK.GroupVersion()} + builder.Register(&TestDefaulter{}, &TestDefaulterList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestDefaulter{}). + WithDefaulter(&TestCustomDefaulter{}). + RecoverPanic(true). + // RecoverPanic defaults to true. + Complete() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + svr := m.GetWebhookServer() + ExpectWithOffset(1, svr).NotTo(BeNil()) + + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"", + "version":"v1", + "kind":"TestDefaulter" + }, + "resource":{ + "group":"", + "version":"v1", + "resource":"testdefaulter" + }, + "namespace":"default", + "operation":"CREATE", + "object":{ + "replica":1, + "panic":true + }, + "oldObject":null + } +}`) + + ctx, cancel := context.WithCancel(specCtx) + cancel() + err = svr.Start(ctx) + if err != nil && !os.IsNotExist(err) { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("sending a request to a mutating webhook path") + path := generateMutatePath(testDefaulterGVK) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w := httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable fields") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":500`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"message":"panic: fake panic test [recovered]`)) + }) + + It("should scaffold a custom validating webhook", func(specCtx SpecContext) { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -222,27 +317,33 @@ func runTests(admissionReviewVersion string) { err = WebhookManagedBy(m). For(&TestValidator{}). + WithValidator(&TestCustomValidator{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + WithContextFunc(func(ctx context.Context, request *http.Request) context.Context { + return context.WithValue(ctx, userAgentCtxKey, request.Header.Get(userAgentHeader)) + }). Complete() ExpectWithOffset(1, err).NotTo(HaveOccurred()) svr := m.GetWebhookServer() ExpectWithOffset(1, svr).NotTo(BeNil()) - reader := strings.NewReader(`{ - "kind":"AdmissionReview", - "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", "request":{ "uid":"07e52e8d-4513-11e9-a716-42010a800270", "kind":{ - "group":"", + "group":"foo.test.org", "version":"v1", "kind":"TestValidator" }, "resource":{ - "group":"", + "group":"foo.test.org", "version":"v1", "resource":"testvalidator" }, "namespace":"default", + "name":"foo", "operation":"UPDATE", "object":{ "replica":1 @@ -252,11 +353,33 @@ func runTests(admissionReviewVersion string) { } } }`) + readerWithCxt := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"foo.test.org", + "version":"v1", + "kind":"TestValidator" + }, + "resource":{ + "group":"foo.test.org", + "version":"v1", + "resource":"testvalidator" + }, + "namespace":"default", + "name":"foo", + "operation":"UPDATE", + "object":{ + "replica":1 + }, + "oldObject":{ + "replica":1 + } + } +}`) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -264,27 +387,42 @@ func runTests(admissionReviewVersion string) { By("sending a request to a mutating webhook path that doesn't exist") path := generateMutatePath(testValidatorGVK) - req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w := httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) By("sending a request to a validating webhook path") path = generateValidatePath(testValidatorGVK) _, err = reader.Seek(0, 0) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w = httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) By("sanity checking the response contains reasonable field") ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":403`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Validating object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testvalidator"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + + By("sending a request to a validating webhook with context header validation") + path = generateValidatePath(testValidatorGVK) + _, err = readerWithCxt.Seek(0, 0) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req = httptest.NewRequest("POST", svcBaseAddr+path, readerWithCxt) + req.Header.Add("Content-Type", "application/json") + req.Header.Add(userAgentHeader, userAgentValue) + w = httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable field") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) }) - It("should scaffold a validating webhook with a custom validator", func() { + It("should scaffold a custom validating webhook with a custom path", func(specCtx SpecContext) { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -295,30 +433,34 @@ func runTests(admissionReviewVersion string) { err = builder.AddToScheme(m.GetScheme()) ExpectWithOffset(1, err).NotTo(HaveOccurred()) + customPath := "/custom-validating-path" err = WebhookManagedBy(m). - WithValidator(&TestCustomValidator{}). For(&TestValidator{}). + WithValidator(&TestCustomValidator{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + WithValidatorCustomPath(customPath). Complete() ExpectWithOffset(1, err).NotTo(HaveOccurred()) svr := m.GetWebhookServer() ExpectWithOffset(1, svr).NotTo(BeNil()) - reader := strings.NewReader(`{ - "kind":"AdmissionReview", - "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", "request":{ "uid":"07e52e8d-4513-11e9-a716-42010a800270", "kind":{ - "group":"", + "group":"foo.test.org", "version":"v1", "kind":"TestValidator" }, "resource":{ - "group":"", + "group":"foo.test.org", "version":"v1", "resource":"testvalidator" }, "namespace":"default", + "name":"foo", "operation":"UPDATE", "object":{ "replica":1 @@ -329,117 +471,104 @@ func runTests(admissionReviewVersion string) { } }`) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) } - By("sending a request to a mutating webhook path that doesn't exist") - path := generateMutatePath(testValidatorGVK) - req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) - req.Header.Add("Content-Type", "application/json") - w := httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) - ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) - - By("sending a request to a validating webhook path") - path = generateValidatePath(testValidatorGVK) + By("sending a request to a valiting webhook path that have been overriten by a custom path") + path, err := generateCustomPath(customPath) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) _, err = reader.Seek(0, 0) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") - w = httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + w := httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) By("sanity checking the response contains reasonable field") ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":403`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Validating object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testvalidator"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + + By("sending a request to a validating webhook path") + path = generateValidatePath(testValidatorGVK) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w = httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) }) - It("should scaffold defaulting and validating webhooks if the type implements both Defaulter and Validator interfaces", func() { + It("should scaffold a custom validating webhook which recovers from panics", func(specCtx SpecContext) { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) ExpectWithOffset(1, err).NotTo(HaveOccurred()) By("registering the type in the Scheme") - builder := scheme.Builder{GroupVersion: testDefaultValidatorGVK.GroupVersion()} - builder.Register(&TestDefaultValidator{}, &TestDefaultValidatorList{}) + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestValidator{}, &TestValidatorList{}) err = builder.AddToScheme(m.GetScheme()) ExpectWithOffset(1, err).NotTo(HaveOccurred()) err = WebhookManagedBy(m). - For(&TestDefaultValidator{}). + For(&TestValidator{}). + WithValidator(&TestCustomValidator{}). + RecoverPanic(true). Complete() ExpectWithOffset(1, err).NotTo(HaveOccurred()) svr := m.GetWebhookServer() ExpectWithOffset(1, svr).NotTo(BeNil()) - reader := strings.NewReader(`{ - "kind":"AdmissionReview", - "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", "request":{ "uid":"07e52e8d-4513-11e9-a716-42010a800270", "kind":{ "group":"", "version":"v1", - "kind":"TestDefaultValidator" + "kind":"TestValidator" }, "resource":{ "group":"", "version":"v1", - "resource":"testdefaultvalidator" + "resource":"testvalidator" }, "namespace":"default", "operation":"CREATE", "object":{ - "replica":1 - }, - "oldObject":null + "replica":2, + "panic":true + } } }`) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) } - By("sending a request to a mutating webhook path") - path := generateMutatePath(testDefaultValidatorGVK) - req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) - req.Header.Add("Content-Type", "application/json") - w := httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) - ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) - By("sanity checking the response contains reasonable field") - ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) - ExpectWithOffset(1, w.Body).To(ContainSubstring(`"patch":`)) - ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) - By("sending a request to a validating webhook path") - path = generateValidatePath(testDefaultValidatorGVK) + path := generateValidatePath(testValidatorGVK) _, err = reader.Seek(0, 0) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") - w = httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + w := httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) By("sanity checking the response contains reasonable field") - ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) - ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":500`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"message":"panic: fake panic test [recovered]`)) }) - It("should scaffold a validating webhook if the type implements the Validator interface to validate deletes", func() { + It("should scaffold a custom validating webhook to validate deletes", func(specCtx SpecContext) { By("creating a controller manager") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) m, err := manager.New(cfg, manager.Options{}) ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -452,14 +581,13 @@ func runTests(admissionReviewVersion string) { err = WebhookManagedBy(m). For(&TestValidator{}). + WithValidator(&TestCustomValidator{}). Complete() ExpectWithOffset(1, err).NotTo(HaveOccurred()) svr := m.GetWebhookServer() ExpectWithOffset(1, svr).NotTo(BeNil()) - reader := strings.NewReader(`{ - "kind":"AdmissionReview", - "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", "request":{ "uid":"07e52e8d-4513-11e9-a716-42010a800270", "kind":{ @@ -482,8 +610,6 @@ func runTests(admissionReviewVersion string) { }`) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -491,18 +617,16 @@ func runTests(admissionReviewVersion string) { By("sending a request to a validating webhook path to check for failed delete") path := generateValidatePath(testValidatorGVK) - req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w := httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) By("sanity checking the response contains reasonable field") ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":403`)) - reader = strings.NewReader(`{ - "kind":"AdmissionReview", - "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + reader = strings.NewReader(admissionReviewGV + admissionReviewVersion + `", "request":{ "uid":"07e52e8d-4513-11e9-a716-42010a800270", "kind":{ @@ -525,25 +649,305 @@ func runTests(admissionReviewVersion string) { }`) By("sending a request to a validating webhook path with correct request") path = generateValidatePath(testValidatorGVK) - req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w = httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable field") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) + }) + + It("should send an error when trying to register a webhook with more than one For", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testDefaultValidatorGVK.GroupVersion()} + builder.Register(&TestDefaulter{}, &TestDefaulterList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestDefaulter{}). + For(&TestDefaulter{}). + Complete() + Expect(err).To(HaveOccurred()) + }) + + It("should scaffold a custom defaulting and validating webhook", func(specCtx SpecContext) { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestDefaultValidator{}, &TestDefaultValidatorList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestDefaultValidator{}). + WithDefaulter(&TestCustomDefaultValidator{}). + WithValidator(&TestCustomDefaultValidator{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + Complete() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + svr := m.GetWebhookServer() + ExpectWithOffset(1, svr).NotTo(BeNil()) + + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"foo.test.org", + "version":"v1", + "kind":"TestDefaultValidator" + }, + "resource":{ + "group":"foo.test.org", + "version":"v1", + "resource":"testdefaultvalidator" + }, + "namespace":"default", + "name":"foo", + "operation":"UPDATE", + "object":{ + "replica":1 + }, + "oldObject":{ + "replica":2 + } + } +}`) + + ctx, cancel := context.WithCancel(specCtx) + cancel() + err = svr.Start(ctx) + if err != nil && !os.IsNotExist(err) { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("sending a request to a mutating webhook path") + path := generateMutatePath(testDefaultValidatorGVK) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w := httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable fields") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"patch":`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Defaulting object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testdefaultvalidator"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + + By("sending a request to a validating webhook path") + path = generateValidatePath(testDefaultValidatorGVK) + _, err = reader.Seek(0, 0) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) req.Header.Add("Content-Type", "application/json") w = httptest.NewRecorder() - svr.WebhookMux.ServeHTTP(w, req) + svr.WebhookMux().ServeHTTP(w, req) ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) By("sanity checking the response contains reasonable field") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":403`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Validating object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testdefaultvalidator"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + }) + + It("should scaffold a custom defaulting and validating webhook with a custom path for each of them", func(specCtx SpecContext) { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestDefaultValidator{}, &TestDefaultValidatorList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + validatingCustomPath := "/custom-validating-path" + defaultingCustomPath := "/custom-defaulting-path" + err = WebhookManagedBy(m). + For(&TestDefaultValidator{}). + WithDefaulter(&TestCustomDefaultValidator{}). + WithValidator(&TestCustomDefaultValidator{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + WithValidatorCustomPath(validatingCustomPath). + WithDefaulterCustomPath(defaultingCustomPath). + Complete() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + svr := m.GetWebhookServer() + ExpectWithOffset(1, svr).NotTo(BeNil()) + + reader := strings.NewReader(admissionReviewGV + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"foo.test.org", + "version":"v1", + "kind":"TestDefaultValidator" + }, + "resource":{ + "group":"foo.test.org", + "version":"v1", + "resource":"testdefaultvalidator" + }, + "namespace":"default", + "name":"foo", + "operation":"UPDATE", + "object":{ + "replica":1 + }, + "oldObject":{ + "replica":2 + } + } +}`) + + ctx, cancel := context.WithCancel(specCtx) + cancel() + err = svr.Start(ctx) + if err != nil && !os.IsNotExist(err) { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("sending a request to a mutating webhook path that have been overriten by the custom path") + path, err := generateCustomPath(defaultingCustomPath) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req := httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w := httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable fields") ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"patch":`)) ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Defaulting object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testdefaultvalidator"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + + By("sending a request to a mutating webhook path") + path = generateMutatePath(testDefaultValidatorGVK) + _, err = reader.Seek(0, 0) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w = httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) + + By("sending a request to a valiting webhook path that have been overriten by a custom path") + path, err = generateCustomPath(validatingCustomPath) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + _, err = reader.Seek(0, 0) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w = httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable field") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":403`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Validating object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testdefaultvalidator"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + + By("sending a request to a validating webhook path") + path = generateValidatePath(testValidatorGVK) + req = httptest.NewRequest("POST", svcBaseAddr+path, reader) + req.Header.Add("Content-Type", "application/json") + w = httptest.NewRecorder() + svr.WebhookMux().ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) + }) + + It("should not scaffold a custom defaulting and a custom validating webhook with the same custom path", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestDefaultValidator{}, &TestDefaultValidatorList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestDefaultValidator{}). + WithDefaulter(&TestCustomDefaultValidator{}). + WithValidator(&TestCustomDefaultValidator{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + WithCustomPath(customPath). + Complete() + ExpectWithOffset(1, err).To(HaveOccurred()) + }) + + It("should not scaffold a custom defaulting when setting a custom path and a defaulting custom path", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestDefaultValidator{}, &TestDefaultValidatorList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestDefaulter{}). + WithDefaulter(&TestCustomDefaulter{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + WithDefaulterCustomPath(customPath). + WithCustomPath(customPath). + Complete() + ExpectWithOffset(1, err).To(HaveOccurred()) + }) + + It("should not scaffold a custom defaulting when setting a custom path and a validating custom path", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestDefaultValidator{}, &TestDefaultValidatorList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestValidator{}). + WithValidator(&TestCustomValidator{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + WithDefaulterCustomPath(customPath). + WithCustomPath(customPath). + Complete() + ExpectWithOffset(1, err).To(HaveOccurred()) }) } // TestDefaulter. var _ runtime.Object = &TestDefaulter{} +const testDefaulterKind = "TestDefaulter" + type TestDefaulter struct { - Replica int `json:"replica,omitempty"` + Replica int `json:"replica,omitempty"` + Panic bool `json:"panic,omitempty"` } -var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestDefaulter"} +var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: testDefaulterKind} func (d *TestDefaulter) GetObjectKind() schema.ObjectKind { return d } func (d *TestDefaulter) DeepCopyObject() runtime.Object { @@ -565,20 +969,17 @@ type TestDefaulterList struct{} func (*TestDefaulterList) GetObjectKind() schema.ObjectKind { return nil } func (*TestDefaulterList) DeepCopyObject() runtime.Object { return nil } -func (d *TestDefaulter) Default() { - if d.Replica < 2 { - d.Replica = 2 - } -} - // TestValidator. var _ runtime.Object = &TestValidator{} +const testValidatorKind = "TestValidator" + type TestValidator struct { - Replica int `json:"replica,omitempty"` + Replica int `json:"replica,omitempty"` + Panic bool `json:"panic,omitempty"` } -var testValidatorGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestValidator"} +var testValidatorGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: testValidatorKind} func (v *TestValidator) GetObjectKind() schema.ObjectKind { return v } func (v *TestValidator) DeepCopyObject() runtime.Object { @@ -600,37 +1001,11 @@ type TestValidatorList struct{} func (*TestValidatorList) GetObjectKind() schema.ObjectKind { return nil } func (*TestValidatorList) DeepCopyObject() runtime.Object { return nil } -var _ admission.Validator = &TestValidator{} - -func (v *TestValidator) ValidateCreate() error { - if v.Replica < 0 { - return errors.New("number of replica should be greater than or equal to 0") - } - return nil -} - -func (v *TestValidator) ValidateUpdate(old runtime.Object) error { - if v.Replica < 0 { - return errors.New("number of replica should be greater than or equal to 0") - } - if oldObj, ok := old.(*TestValidator); !ok { - return fmt.Errorf("the old object is expected to be %T", oldObj) - } else if v.Replica < oldObj.Replica { - return fmt.Errorf("new replica %v should not be fewer than old replica %v", v.Replica, oldObj.Replica) - } - return nil -} - -func (v *TestValidator) ValidateDelete() error { - if v.Replica > 0 { - return errors.New("number of replica should be less than or equal to 0 to delete") - } - return nil -} - // TestDefaultValidator. var _ runtime.Object = &TestDefaultValidator{} +const testDefaultValidatorKind = "TestDefaultValidator" + type TestDefaultValidator struct { metav1.TypeMeta metav1.ObjectMeta @@ -660,41 +1035,119 @@ type TestDefaultValidatorList struct{} func (*TestDefaultValidatorList) GetObjectKind() schema.ObjectKind { return nil } func (*TestDefaultValidatorList) DeepCopyObject() runtime.Object { return nil } -func (dv *TestDefaultValidator) Default() { - if dv.Replica < 2 { - dv.Replica = 2 +// TestCustomDefaulter. +type TestCustomDefaulter struct{} + +func (*TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) error { + logf.FromContext(ctx).Info("Defaulting object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaulterKind { + return fmt.Errorf("expected Kind TestDefaulter got %q", req.Kind.Kind) } -} -var _ admission.Validator = &TestDefaultValidator{} + d := obj.(*TestDefaulter) //nolint:ifshort + if d.Panic { + panic("fake panic test") + } -func (dv *TestDefaultValidator) ValidateCreate() error { - if dv.Replica < 0 { - return errors.New("number of replica should be greater than or equal to 0") + if d.Replica < 2 { + d.Replica = 2 } + return nil } -func (dv *TestDefaultValidator) ValidateUpdate(old runtime.Object) error { - if dv.Replica < 0 { - return errors.New("number of replica should be greater than or equal to 0") +var _ admission.CustomDefaulter = &TestCustomDefaulter{} + +// TestCustomValidator. + +type TestCustomValidator struct{} + +func (*TestCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) } - return nil + if req.Kind.Kind != testValidatorKind { + return nil, fmt.Errorf("expected Kind TestValidator got %q", req.Kind.Kind) + } + + v := obj.(*TestValidator) //nolint:ifshort + if v.Panic { + panic("fake panic test") + } + if v.Replica < 0 { + return nil, errors.New("number of replica should be greater than or equal to 0") + } + + return nil, nil } -func (dv *TestDefaultValidator) ValidateDelete() error { - if dv.Replica > 0 { - return errors.New("number of replica should be less than or equal to 0 to delete") +func (*TestCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) } - return nil + if req.Kind.Kind != testValidatorKind { + return nil, fmt.Errorf("expected Kind TestValidator got %q", req.Kind.Kind) + } + + v := newObj.(*TestValidator) + old := oldObj.(*TestValidator) + if v.Replica < 0 { + return nil, errors.New("number of replica should be greater than or equal to 0") + } + if v.Replica < old.Replica { + return nil, fmt.Errorf("new replica %v should not be fewer than old replica %v", v.Replica, old.Replica) + } + + userAgent, ok := ctx.Value(userAgentCtxKey).(string) + if ok && userAgent != userAgentValue { + return nil, fmt.Errorf("expected %s value is %q in TestCustomValidator got %q", userAgentCtxKey, userAgentValue, userAgent) + } + + return nil, nil } -// TestCustomDefaulter. +func (*TestCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testValidatorKind { + return nil, fmt.Errorf("expected Kind TestValidator got %q", req.Kind.Kind) + } -type TestCustomDefaulter struct{} + v := obj.(*TestValidator) //nolint:ifshort + if v.Replica > 0 { + return nil, errors.New("number of replica should be less than or equal to 0 to delete") + } + return nil, nil +} + +var _ admission.CustomValidator = &TestCustomValidator{} + +// TestCustomDefaultValidator for default +type TestCustomDefaultValidator struct{} + +func (*TestCustomDefaultValidator) Default(ctx context.Context, obj runtime.Object) error { + logf.FromContext(ctx).Info("Defaulting object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } + + d := obj.(*TestDefaultValidator) //nolint:ifshort -func (*TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) error { - d := obj.(*TestDefaulter) //nolint:ifshort if d.Replica < 2 { d.Replica = 2 } @@ -703,36 +1156,61 @@ func (*TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) err var _ admission.CustomDefaulter = &TestCustomDefaulter{} -// TestCustomValidator. +// TestCustomDefaultValidator for validation -type TestCustomValidator struct{} +func (*TestCustomDefaultValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return nil, fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } -func (*TestCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error { - v := obj.(*TestValidator) //nolint:ifshort + v := obj.(*TestDefaultValidator) //nolint:ifshort if v.Replica < 0 { - return errors.New("number of replica should be greater than or equal to 0") + return nil, errors.New("number of replica should be greater than or equal to 0") } - return nil + return nil, nil } -func (*TestCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { - v := newObj.(*TestValidator) - old := oldObj.(*TestValidator) //nolint:ifshort +func (*TestCustomDefaultValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return nil, fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } + + v := newObj.(*TestDefaultValidator) + old := oldObj.(*TestDefaultValidator) if v.Replica < 0 { - return errors.New("number of replica should be greater than or equal to 0") + return nil, errors.New("number of replica should be greater than or equal to 0") } if v.Replica < old.Replica { - return fmt.Errorf("new replica %v should not be fewer than old replica %v", v.Replica, old.Replica) + return nil, fmt.Errorf("new replica %v should not be fewer than old replica %v", v.Replica, old.Replica) } - return nil + return nil, nil } -func (*TestCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error { - v := obj.(*TestValidator) //nolint:ifshort +func (*TestCustomDefaultValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return nil, fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } + + v := obj.(*TestDefaultValidator) //nolint:ifshort if v.Replica > 0 { - return errors.New("number of replica should be less than or equal to 0 to delete") + return nil, errors.New("number of replica should be less than or equal to 0 to delete") } - return nil + return nil, nil } var _ admission.CustomValidator = &TestCustomValidator{} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index f89800ca2f..a7e491855a 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -19,30 +19,55 @@ package cache import ( "context" "fmt" + "maps" + "net/http" + "slices" + "sort" "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/cache/internal" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) -var log = logf.RuntimeLog.WithName("object-cache") +var ( + defaultSyncPeriod = 10 * time.Hour +) + +// InformerGetOptions defines the behavior of how informers are retrieved. +type InformerGetOptions internal.GetOptions + +// InformerGetOption defines an option that alters the behavior of how informers are retrieved. +type InformerGetOption func(*InformerGetOptions) + +// BlockUntilSynced determines whether a get request for an informer should block +// until the informer's cache has synced. +func BlockUntilSynced(shouldBlock bool) InformerGetOption { + return func(opts *InformerGetOptions) { + opts.BlockUntilSynced = &shouldBlock + } +} // Cache knows how to load Kubernetes objects, fetch informers to request // to receive events for Kubernetes objects (at a low-level), // and add indices to fields on the objects stored in the cache. type Cache interface { - // Cache acts as a client to objects stored in the cache. + // Reader acts as a client to objects stored in the cache. client.Reader - // Cache loads informers and adds field indices. + // Informers loads informers and adds field indices. Informers } @@ -52,123 +77,397 @@ type Cache interface { type Informers interface { // GetInformer fetches or constructs an informer for the given object that corresponds to a single // API kind and resource. - GetInformer(ctx context.Context, obj client.Object) (Informer, error) + GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) // GetInformerForKind is similar to GetInformer, except that it takes a group-version-kind, instead // of the underlying object. - GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) + GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) + + // RemoveInformer removes an informer entry and stops it if it was running. + RemoveInformer(ctx context.Context, obj client.Object) error // Start runs all the informers known to this cache until the context is closed. // It blocks. Start(ctx context.Context) error - // WaitForCacheSync waits for all the caches to sync. Returns false if it could not sync a cache. + // WaitForCacheSync waits for all the caches to sync. Returns false if it could not sync a cache. WaitForCacheSync(ctx context.Context) bool - // Informers knows how to add indices to the caches (informers) that it manages. + // FieldIndexer adds indices to the managed informers. client.FieldIndexer } -// Informer - informer allows you interact with the underlying informer. +// Informer allows you to interact with the underlying informer. type Informer interface { // AddEventHandler adds an event handler to the shared informer using the shared informer's resync - // period. Events to a single handler are delivered sequentially, but there is no coordination + // period. Events to a single handler are delivered sequentially, but there is no coordination // between different handlers. - AddEventHandler(handler toolscache.ResourceEventHandler) + // It returns a registration handle for the handler that can be used to remove + // the handler again and an error if the handler cannot be added. + AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) + // AddEventHandlerWithResyncPeriod adds an event handler to the shared informer using the - // specified resync period. Events to a single handler are delivered sequentially, but there is + // specified resync period. Events to a single handler are delivered sequentially, but there is // no coordination between different handlers. - AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) - // AddIndexers adds more indexers to this store. If you call this after you already have data - // in the store, the results are undefined. + // It returns a registration handle for the handler that can be used to remove + // the handler again and an error if the handler cannot be added. + AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) + + // AddEventHandlerWithOptions is a variant of AddEventHandlerWithResyncPeriod where + // all optional parameters are passed in as a struct. + AddEventHandlerWithOptions(handler toolscache.ResourceEventHandler, options toolscache.HandlerOptions) (toolscache.ResourceEventHandlerRegistration, error) + + // RemoveEventHandler removes a previously added event handler given by + // its registration handle. + // This function is guaranteed to be idempotent and thread-safe. + RemoveEventHandler(handle toolscache.ResourceEventHandlerRegistration) error + + // AddIndexers adds indexers to this store. It is valid to add indexers + // after an informer was started. AddIndexers(indexers toolscache.Indexers) error + // HasSynced return true if the informers underlying store has synced. HasSynced() bool + // IsStopped returns true if the informer has been stopped. + IsStopped() bool } -// SelectorsByObject associate a client.Object's GVK to a field/label selector. -type SelectorsByObject map[client.Object]internal.Selector +// AllNamespaces should be used as the map key to deliminate namespace settings +// that apply to all namespaces that themselves do not have explicit settings. +const AllNamespaces = metav1.NamespaceAll -// Options are the optional arguments for creating a new InformersMap object. +// Options are the optional arguments for creating a new Cache object. type Options struct { + // HTTPClient is the http client to use for the REST client + HTTPClient *http.Client + // Scheme is the scheme to use for mapping objects to GroupVersionKinds Scheme *runtime.Scheme // Mapper is the RESTMapper to use for mapping GroupVersionKinds to Resources Mapper meta.RESTMapper - // Resync is the base frequency the informers are resynced. - // Defaults to defaultResyncTime. - // A 10 percent jitter will be added to the Resync period between informers - // So that all informers will not send list requests simultaneously. - Resync *time.Duration - - // Namespace restricts the cache's ListWatch to the desired namespace - // Default watches all namespaces - Namespace string - - // SelectorsByObject restricts the cache's ListWatch to the desired - // fields per GVK at the specified object, the map's value must implement - // Selector [1] using for example a Set [2] - // [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Selector - // [2] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Set - SelectorsByObject SelectorsByObject + // SyncPeriod determines the minimum frequency at which watched resources are + // reconciled. A lower period will correct entropy more quickly, but reduce + // responsiveness to change if there are many watched resources. Change this + // value only if you know what you are doing. Defaults to 10 hours if unset. + // there will a 10 percent jitter between the SyncPeriod of all controllers + // so that all controllers will not send list requests simultaneously. + // + // This applies to all controllers. + // + // A period sync happens for two reasons: + // 1. To insure against a bug in the controller that causes an object to not + // be requeued, when it otherwise should be requeued. + // 2. To insure against an unknown bug in controller-runtime, or its dependencies, + // that causes an object to not be requeued, when it otherwise should be + // requeued, or to be removed from the queue, when it otherwise should not + // be removed. + // + // If you want + // 1. to insure against missed watch events, or + // 2. to poll services that cannot be watched, + // then we recommend that, instead of changing the default period, the + // controller requeue, with a constant duration `t`, whenever the controller + // is "done" with an object, and would otherwise not requeue it, i.e., we + // recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`, + // instead of `reconcile.Result{}`. + // + // SyncPeriod will locally trigger an artificial Update event with the same + // object in both ObjectOld and ObjectNew for everything that is in the + // cache. + // + // Predicates or Handlers that expect ObjectOld and ObjectNew to be different + // (such as GenerationChangedPredicate) will filter out this event, preventing + // it from triggering a reconciliation. + // SyncPeriod does not sync between the local cache and the server. + SyncPeriod *time.Duration + + // ReaderFailOnMissingInformer configures the cache to return a ErrResourceNotCached error when a user + // requests, using Get() and List(), a resource the cache does not already have an informer for. + // + // This error is distinct from an errors.NotFound. + // + // Defaults to false, which means that the cache will start a new informer + // for every new requested resource. + ReaderFailOnMissingInformer bool + + // DefaultNamespaces maps namespace names to cache configs. If set, only + // the namespaces in here will be watched and it will by used to default + // ByObject.Namespaces for all objects if that is nil. + // + // It is possible to have specific Config for just some namespaces + // but cache all namespaces by using the AllNamespaces const as the map key. + // This will then include all namespaces that do not have a more specific + // setting. + // + // The options in the Config that are nil will be defaulted from + // the respective Default* settings. + DefaultNamespaces map[string]Config + + // DefaultLabelSelector will be used as a label selector for all objects + // unless there is already one set in ByObject or DefaultNamespaces. + DefaultLabelSelector labels.Selector + + // DefaultFieldSelector will be used as a field selector for all object types + // unless there is already one set in ByObject or DefaultNamespaces. + DefaultFieldSelector fields.Selector + + // DefaultTransform will be used as transform for all object types + // unless there is already one set in ByObject or DefaultNamespaces. + // + // A typical usecase for this is to use TransformStripManagedFields + // to reduce the caches memory usage. + DefaultTransform toolscache.TransformFunc + + // DefaultWatchErrorHandler will be used to set the WatchErrorHandler which is called + // whenever ListAndWatch drops the connection with an error. + // + // After calling this handler, the informer will backoff and retry. + DefaultWatchErrorHandler toolscache.WatchErrorHandlerWithContext + + // DefaultUnsafeDisableDeepCopy is the default for UnsafeDisableDeepCopy + // for everything that doesn't specify this. + // + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + // + // This will be used for all object types, unless it is set in ByObject or + // DefaultNamespaces. + DefaultUnsafeDisableDeepCopy *bool + + // DefaultEnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // This will be used for all object types, unless it is set in ByObject or + // DefaultNamespaces. + // + // Defaults to true. + DefaultEnableWatchBookmarks *bool + + // ByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object. + // If unset, this will fall through to the Default* settings. + ByObject map[client.Object]ByObject + + // NewInformer allows overriding of NewSharedIndexInformer, for example for testing + // or if someone wants to write their own Informer. + NewInformer func(toolscache.ListerWatcher, runtime.Object, time.Duration, toolscache.Indexers) toolscache.SharedIndexInformer +} - // UnsafeDisableDeepCopyByObject indicates not to deep copy objects during get or +// ByObject offers more fine-grained control over the cache's ListWatch by object. +type ByObject struct { + // Namespaces maps a namespace name to cache configs. If set, only the + // namespaces in this map will be cached. + // + // Settings in the map value that are unset will be defaulted. + // Use an empty value for the specific setting to prevent that. + // + // It is possible to have specific Config for just some namespaces + // but cache all namespaces by using the AllNamespaces const as the map key. + // This will then include all namespaces that do not have a more specific + // setting. + // + // A nil map allows to default this to the cache's DefaultNamespaces setting. + // An empty map prevents this and means that all namespaces will be cached. + // + // The defaulting follows the following precedence order: + // 1. ByObject + // 2. DefaultNamespaces[namespace] + // 3. Default* + // + // This must be unset for cluster-scoped objects. + Namespaces map[string]Config + + // Label represents a label selector for the object. + Label labels.Selector + + // Field represents a field selector for the object. + Field fields.Selector + + // Transform is a transformer function for the object which gets applied + // when objects of the transformation are about to be committed to the cache. + // + // This function is called both for new objects to enter the cache, + // and for updated objects. + Transform toolscache.TransformFunc + + // UnsafeDisableDeepCopy indicates not to deep copy objects during get or // list objects per GVK at the specified object. // Be very careful with this, when enabled you must DeepCopy any object before mutating it, // otherwise you will mutate the object in the cache. - UnsafeDisableDeepCopyByObject DisableDeepCopyByObject + UnsafeDisableDeepCopy *bool + + // EnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // Defaults to true. + EnableWatchBookmarks *bool +} + +// Config describes all potential options for a given watch. +type Config struct { + // LabelSelector specifies a label selector. A nil value allows to + // default this. + // + // Set to labels.Everything() if you don't want this defaulted. + LabelSelector labels.Selector + + // FieldSelector specifics a field selector. A nil value allows to + // default this. + // + // Set to fields.Everything() if you don't want this defaulted. + FieldSelector fields.Selector + + // Transform specifies a transform func. A nil value allows to default + // this. + // + // Set to an empty func to prevent this: + // func(in interface{}) (interface{}, error) { return in, nil } + Transform toolscache.TransformFunc + + // UnsafeDisableDeepCopy specifies if List and Get requests against the + // cache should not DeepCopy. A nil value allows to default this. + UnsafeDisableDeepCopy *bool + + // EnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // Defaults to true. + EnableWatchBookmarks *bool } -var defaultResyncTime = 10 * time.Hour +// NewCacheFunc - Function for creating a new cache from the options and a rest config. +type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error) // New initializes and returns a new Cache. -func New(config *rest.Config, opts Options) (Cache, error) { - opts, err := defaultOpts(config, opts) +func New(cfg *rest.Config, opts Options) (Cache, error) { + opts, err := defaultOpts(cfg, opts) if err != nil { return nil, err } - selectorsByGVK, err := convertToSelectorsByGVK(opts.SelectorsByObject, opts.Scheme) - if err != nil { - return nil, err + + newCacheFunc := newCache(cfg, opts) + + var defaultCache Cache + if len(opts.DefaultNamespaces) > 0 { + defaultConfig := optionDefaultsToConfig(&opts) + defaultCache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, opts.DefaultNamespaces, &defaultConfig) + } else { + defaultCache = newCacheFunc(optionDefaultsToConfig(&opts), corev1.NamespaceAll) } - disableDeepCopyByGVK, err := convertToDisableDeepCopyByGVK(opts.UnsafeDisableDeepCopyByObject, opts.Scheme) - if err != nil { - return nil, err + + if len(opts.ByObject) == 0 { + return defaultCache, nil + } + + delegating := &delegatingByGVKCache{ + scheme: opts.Scheme, + caches: make(map[schema.GroupVersionKind]Cache, len(opts.ByObject)), + defaultCache: defaultCache, } - im := internal.NewInformersMap(config, opts.Scheme, opts.Mapper, *opts.Resync, opts.Namespace, selectorsByGVK, disableDeepCopyByGVK) - return &informerCache{InformersMap: im}, nil -} - -// BuilderWithOptions returns a Cache constructor that will build the a cache -// honoring the options argument, this is useful to specify options like -// SelectorsByObject -// WARNING: if SelectorsByObject is specified. filtered out resources are not -// returned. -// WARNING: if UnsafeDisableDeepCopy is enabled, you must DeepCopy any object -// returned from cache get/list before mutating it. -func BuilderWithOptions(options Options) NewCacheFunc { - return func(config *rest.Config, opts Options) (Cache, error) { - if opts.Scheme == nil { - opts.Scheme = options.Scheme + + for obj, config := range opts.ByObject { + gvk, err := apiutil.GVKForObject(obj, opts.Scheme) + if err != nil { + return nil, fmt.Errorf("failed to get GVK for type %T: %w", obj, err) } - if opts.Mapper == nil { - opts.Mapper = options.Mapper + var cache Cache + if len(config.Namespaces) > 0 { + cache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, config.Namespaces, nil) + } else { + cache = newCacheFunc(byObjectToConfig(config), corev1.NamespaceAll) } - if opts.Resync == nil { - opts.Resync = options.Resync + delegating.caches[gvk] = cache + } + + return delegating, nil +} + +// TransformStripManagedFields strips the managed fields of an object before it is committed to the cache. +// If you are not explicitly accessing managedFields from your code, setting this as `DefaultTransform` +// on the cache can lead to a significant reduction in memory usage. +func TransformStripManagedFields() toolscache.TransformFunc { + return func(in any) (any, error) { + // Nilcheck managed fields to avoid hitting https://github.com/kubernetes/kubernetes/issues/124337 + if obj, err := meta.Accessor(in); err == nil && obj.GetManagedFields() != nil { + obj.SetManagedFields(nil) } - if opts.Namespace == "" { - opts.Namespace = options.Namespace + + return in, nil + } +} + +func optionDefaultsToConfig(opts *Options) Config { + return Config{ + LabelSelector: opts.DefaultLabelSelector, + FieldSelector: opts.DefaultFieldSelector, + Transform: opts.DefaultTransform, + UnsafeDisableDeepCopy: opts.DefaultUnsafeDisableDeepCopy, + EnableWatchBookmarks: opts.DefaultEnableWatchBookmarks, + } +} + +func byObjectToConfig(byObject ByObject) Config { + return Config{ + LabelSelector: byObject.Label, + FieldSelector: byObject.Field, + Transform: byObject.Transform, + UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy, + EnableWatchBookmarks: byObject.EnableWatchBookmarks, + } +} + +type newCacheFunc func(config Config, namespace string) Cache + +func newCache(restConfig *rest.Config, opts Options) newCacheFunc { + return func(config Config, namespace string) Cache { + return &informerCache{ + scheme: opts.Scheme, + Informers: internal.NewInformers(restConfig, &internal.InformersOpts{ + HTTPClient: opts.HTTPClient, + Scheme: opts.Scheme, + Mapper: opts.Mapper, + ResyncPeriod: *opts.SyncPeriod, + Namespace: namespace, + Selector: internal.Selector{ + Label: config.LabelSelector, + Field: config.FieldSelector, + }, + Transform: config.Transform, + WatchErrorHandler: opts.DefaultWatchErrorHandler, + UnsafeDisableDeepCopy: ptr.Deref(config.UnsafeDisableDeepCopy, false), + EnableWatchBookmarks: ptr.Deref(config.EnableWatchBookmarks, true), + NewInformer: opts.NewInformer, + }), + readerFailOnMissingInformer: opts.ReaderFailOnMissingInformer, } - opts.SelectorsByObject = options.SelectorsByObject - opts.UnsafeDisableDeepCopyByObject = options.UnsafeDisableDeepCopyByObject - return New(config, opts) } } func defaultOpts(config *rest.Config, opts Options) (Options, error) { + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // Use the rest HTTP client for the provided config if unset + if opts.HTTPClient == nil { + var err error + opts.HTTPClient, err = rest.HTTPClientFor(config) + if err != nil { + return Options{}, fmt.Errorf("could not create HTTP client from config: %w", err) + } + } + // Use the default Kubernetes Scheme if unset if opts.Scheme == nil { opts.Scheme = scheme.Scheme @@ -177,55 +476,126 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { // Construct a new Mapper if unset if opts.Mapper == nil { var err error - opts.Mapper, err = apiutil.NewDiscoveryRESTMapper(config) + opts.Mapper, err = apiutil.NewDynamicRESTMapper(config, opts.HTTPClient) if err != nil { - log.WithName("setup").Error(err, "Failed to get API Group-Resources") - return opts, fmt.Errorf("could not create RESTMapper from config") + return Options{}, fmt.Errorf("could not create RESTMapper from config: %w", err) + } + } + + opts.ByObject = maps.Clone(opts.ByObject) + opts.DefaultNamespaces = maps.Clone(opts.DefaultNamespaces) + for obj, byObject := range opts.ByObject { + isNamespaced, err := apiutil.IsObjectNamespaced(obj, opts.Scheme, opts.Mapper) + if err != nil { + return opts, fmt.Errorf("failed to determine if %T is namespaced: %w", obj, err) + } + if !isNamespaced && byObject.Namespaces != nil { + return opts, fmt.Errorf("type %T is not namespaced, but its ByObject.Namespaces setting is not nil", obj) + } + + if isNamespaced && byObject.Namespaces == nil { + byObject.Namespaces = maps.Clone(opts.DefaultNamespaces) + } else { + byObject.Namespaces = maps.Clone(byObject.Namespaces) + } + + // Default the namespace-level configs first, because they need to use the undefaulted type-level config + // to be able to potentially fall through to settings from DefaultNamespaces. + for namespace, config := range byObject.Namespaces { + // 1. Default from the undefaulted type-level config + config = defaultConfig(config, byObjectToConfig(byObject)) + // 2. Default from the namespace-level config. This was defaulted from the global default config earlier, but + // might not have an entry for the current namespace. + if defaultNamespaceSettings, hasDefaultNamespace := opts.DefaultNamespaces[namespace]; hasDefaultNamespace { + config = defaultConfig(config, defaultNamespaceSettings) + } + + // 3. Default from the global defaults + config = defaultConfig(config, optionDefaultsToConfig(&opts)) + + if namespace == metav1.NamespaceAll { + config.FieldSelector = fields.AndSelectors( + appendIfNotNil( + namespaceAllSelector(slices.Collect(maps.Keys(byObject.Namespaces))), + config.FieldSelector, + )..., + ) + } + + byObject.Namespaces[namespace] = config + } + + // Only default ByObject iself if it isn't namespaced or has no namespaces configured, as only + // then any of this will be honored. + if !isNamespaced || len(byObject.Namespaces) == 0 { + defaultedConfig := defaultConfig(byObjectToConfig(byObject), optionDefaultsToConfig(&opts)) + byObject.Label = defaultedConfig.LabelSelector + byObject.Field = defaultedConfig.FieldSelector + byObject.Transform = defaultedConfig.Transform + byObject.UnsafeDisableDeepCopy = defaultedConfig.UnsafeDisableDeepCopy + byObject.EnableWatchBookmarks = defaultedConfig.EnableWatchBookmarks } + + opts.ByObject[obj] = byObject + } + + // Default namespaces after byObject has been defaulted, otherwise a namespace without selectors + // will get the `Default` selectors, then get copied to byObject and then not get defaulted from + // byObject, as it already has selectors. + for namespace, cfg := range opts.DefaultNamespaces { + cfg = defaultConfig(cfg, optionDefaultsToConfig(&opts)) + if namespace == metav1.NamespaceAll { + cfg.FieldSelector = fields.AndSelectors( + appendIfNotNil( + namespaceAllSelector(slices.Collect(maps.Keys(opts.DefaultNamespaces))), + cfg.FieldSelector, + )..., + ) + } + opts.DefaultNamespaces[namespace] = cfg } // Default the resync period to 10 hours if unset - if opts.Resync == nil { - opts.Resync = &defaultResyncTime + if opts.SyncPeriod == nil { + opts.SyncPeriod = &defaultSyncPeriod } return opts, nil } -func convertToSelectorsByGVK(selectorsByObject SelectorsByObject, scheme *runtime.Scheme) (internal.SelectorsByGVK, error) { - selectorsByGVK := internal.SelectorsByGVK{} - for object, selector := range selectorsByObject { - gvk, err := apiutil.GVKForObject(object, scheme) - if err != nil { - return nil, err - } - selectorsByGVK[gvk] = selector +func defaultConfig(toDefault, defaultFrom Config) Config { + if toDefault.LabelSelector == nil { + toDefault.LabelSelector = defaultFrom.LabelSelector + } + if toDefault.FieldSelector == nil { + toDefault.FieldSelector = defaultFrom.FieldSelector + } + if toDefault.Transform == nil { + toDefault.Transform = defaultFrom.Transform + } + if toDefault.UnsafeDisableDeepCopy == nil { + toDefault.UnsafeDisableDeepCopy = defaultFrom.UnsafeDisableDeepCopy + } + if toDefault.EnableWatchBookmarks == nil { + toDefault.EnableWatchBookmarks = defaultFrom.EnableWatchBookmarks } - return selectorsByGVK, nil + return toDefault } -// DisableDeepCopyByObject associate a client.Object's GVK to disable DeepCopy during get or list from cache. -type DisableDeepCopyByObject map[client.Object]bool - -var _ client.Object = &ObjectAll{} +func namespaceAllSelector(namespaces []string) []fields.Selector { + selectors := make([]fields.Selector, 0, len(namespaces)-1) + sort.Strings(namespaces) + for _, namespace := range namespaces { + if namespace != metav1.NamespaceAll { + selectors = append(selectors, fields.OneTermNotEqualSelector("metadata.namespace", namespace)) + } + } -// ObjectAll is the argument to represent all objects' types. -type ObjectAll struct { - client.Object + return selectors } -func convertToDisableDeepCopyByGVK(disableDeepCopyByObject DisableDeepCopyByObject, scheme *runtime.Scheme) (internal.DisableDeepCopyByGVK, error) { - disableDeepCopyByGVK := internal.DisableDeepCopyByGVK{} - for obj, disable := range disableDeepCopyByObject { - switch obj.(type) { - case ObjectAll, *ObjectAll: - disableDeepCopyByGVK[internal.GroupVersionKindAll] = disable - default: - gvk, err := apiutil.GVKForObject(obj, scheme) - if err != nil { - return nil, err - } - disableDeepCopyByGVK[gvk] = disable - } +func appendIfNotNil[T comparable](a []T, b T) []T { + if b != *new(T) { + return append(a, b) } - return disableDeepCopyByGVK, nil + return a } diff --git a/pkg/cache/cache_suite_test.go b/pkg/cache/cache_suite_test.go index 2517777d39..a9a5152ce8 100644 --- a/pkg/cache/cache_suite_test.go +++ b/pkg/cache/cache_suite_test.go @@ -19,20 +19,18 @@ package cache_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Cache Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Cache Suite") } var testenv *envtest.Environment @@ -50,7 +48,7 @@ var _ = BeforeSuite(func() { clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 9ae290636c..7748e2e317 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -18,37 +18,44 @@ package cache_test import ( "context" + "errors" "fmt" "reflect" "sort" + "strconv" + "strings" + "time" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" kcache "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" ) const testNodeOne = "test-node-1" +const testNodeTwo = "test-node-2" const testNamespaceOne = "test-namespace-1" const testNamespaceTwo = "test-namespace-2" const testNamespaceThree = "test-namespace-3" // TODO(community): Pull these helper functions into testenv. // Restart policy is included to allow indexing on that field. -func createPodWithLabels(name, namespace string, restartPolicy corev1.RestartPolicy, labels map[string]string) client.Object { +func createPodWithLabels(ctx context.Context, name, namespace string, restartPolicy corev1.RestartPolicy, labels map[string]string) client.Object { three := int64(3) if labels == nil { labels = map[string]string{} @@ -68,37 +75,534 @@ func createPodWithLabels(name, namespace string, restartPolicy corev1.RestartPol } cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = cl.Create(context.Background(), pod) + err = cl.Create(ctx, pod) Expect(err).NotTo(HaveOccurred()) return pod } -func createPod(name, namespace string, restartPolicy corev1.RestartPolicy) client.Object { - return createPodWithLabels(name, namespace, restartPolicy, nil) +func createSvc(ctx context.Context, name, namespace string, cl client.Client) client.Object { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 1}}, + }, + } + err := cl.Create(ctx, svc) + Expect(err).NotTo(HaveOccurred()) + return svc +} + +func createSA(ctx context.Context, name, namespace string, cl client.Client) client.Object { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + err := cl.Create(ctx, sa) + Expect(err).NotTo(HaveOccurred()) + return sa +} + +func createPod(ctx context.Context, name, namespace string, restartPolicy corev1.RestartPolicy) client.Object { + return createPodWithLabels(ctx, name, namespace, restartPolicy, nil) } -func deletePod(pod client.Object) { +func deletePod(ctx context.Context, pod client.Object) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = cl.Delete(context.Background(), pod) + err = cl.Delete(ctx, pod) Expect(err).NotTo(HaveOccurred()) } var _ = Describe("Informer Cache", func() { CacheTest(cache.New, cache.Options{}) + NonBlockingGetTest(cache.New, cache.Options{}) +}) + +var _ = Describe("Informer Cache with ReaderFailOnMissingInformer", func() { + CacheTestReaderFailOnMissingInformer(cache.New, cache.Options{ReaderFailOnMissingInformer: true}) }) + var _ = Describe("Multi-Namespace Informer Cache", func() { - CacheTest(cache.MultiNamespacedCacheBuilder([]string{testNamespaceOne, testNamespaceTwo, "default"}), cache.Options{}) + CacheTest(cache.New, cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + cache.AllNamespaces: {FieldSelector: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne)}, + testNamespaceTwo: {}, + "default": {}, + }, + }) + NonBlockingGetTest(cache.New, cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + cache.AllNamespaces: {FieldSelector: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne)}, + testNamespaceTwo: {}, + "default": {}, + }, + }) +}) + +var _ = Describe("Informer Cache without global DeepCopy", func() { + CacheTest(cache.New, cache.Options{ + DefaultUnsafeDisableDeepCopy: ptr.To(true), + }) + NonBlockingGetTest(cache.New, cache.Options{ + DefaultUnsafeDisableDeepCopy: ptr.To(true), + }) +}) + +var _ = Describe("Cache with transformers", func() { + var ( + informerCache cache.Cache + informerCacheCancel context.CancelFunc + knownPod1 client.Object + knownPod2 client.Object + knownPod3 client.Object + knownPod4 client.Object + knownPod5 client.Object + knownPod6 client.Object + ) + + getTransformValue := func(obj client.Object) string { + accessor, err := meta.Accessor(obj) + if err == nil { + annotations := accessor.GetAnnotations() + if val, exists := annotations["transformed"]; exists { + return val + } + } + return "" + } + + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo + Expect(cfg).NotTo(BeNil()) + + By("creating three pods") + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + err = ensureNode(ctx, testNodeOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceTwo, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceThree, cl) + Expect(err).NotTo(HaveOccurred()) + // Includes restart policy since these objects are indexed on this field. + knownPod1 = createPod(ctx, "test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) + knownPod2 = createPod(ctx, "test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod3 = createPodWithLabels(ctx, "test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) + knownPod4 = createPodWithLabels(ctx, "test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) + knownPod5 = createPod(ctx, "test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) + knownPod6 = createPod(ctx, "test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) + + podGVK := schema.GroupVersionKind{ + Kind: "Pod", + Version: "v1", + } + + knownPod1.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod2.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod3.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod4.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod5.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod6.GetObjectKind().SetGroupVersionKind(podGVK) + + By("creating the informer cache") + informerCache, err = cache.New(cfg, cache.Options{ + DefaultTransform: func(i interface{}) (interface{}, error) { + obj := i.(runtime.Object) + Expect(obj).NotTo(BeNil()) + + accessor, err := meta.Accessor(obj) + Expect(err).ToNot(HaveOccurred()) + annotations := accessor.GetAnnotations() + + if _, exists := annotations["transformed"]; exists { + // Avoid performing transformation multiple times. + return i, nil + } + + if annotations == nil { + annotations = make(map[string]string) + } + annotations["transformed"] = "default" + accessor.SetAnnotations(annotations) + return i, nil + }, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Transform: func(i interface{}) (interface{}, error) { + obj := i.(runtime.Object) + Expect(obj).NotTo(BeNil()) + accessor, err := meta.Accessor(obj) + Expect(err).ToNot(HaveOccurred()) + + annotations := accessor.GetAnnotations() + if _, exists := annotations["transformed"]; exists { + // Avoid performing transformation multiple times. + return i, nil + } + + if annotations == nil { + annotations = make(map[string]string) + } + annotations["transformed"] = "explicit" + accessor.SetAnnotations(annotations) + return i, nil + }, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) + }) + + AfterEach(func(ctx SpecContext) { + By("cleaning up created pods") + deletePod(ctx, knownPod1) + deletePod(ctx, knownPod2) + deletePod(ctx, knownPod3) + deletePod(ctx, knownPod4) + deletePod(ctx, knownPod5) + deletePod(ctx, knownPod6) + + informerCacheCancel() + }) + + Context("with structured objects", func() { + It("should apply transformers to explicitly specified GVKS", func(ctx SpecContext) { + By("listing pods") + out := corev1.PodList{} + Expect(informerCache.List(ctx, &out)).To(Succeed()) + + By("verifying that the returned pods were transformed") + for i := 0; i < len(out.Items); i++ { + Expect(getTransformValue(&out.Items[i])).To(BeIdenticalTo("explicit")) + } + }) + + It("should apply default transformer to objects when none is specified", func(ctx SpecContext) { + By("getting the Kubernetes service") + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) + + By("verifying that the returned service was transformed") + Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) + }) + }) + + Context("with unstructured objects", func() { + It("should apply transformers to explicitly specified GVKS", func(ctx SpecContext) { + By("listing pods") + out := unstructured.UnstructuredList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + Expect(informerCache.List(ctx, &out)).To(Succeed()) + + By("verifying that the returned pods were transformed") + for i := 0; i < len(out.Items); i++ { + Expect(getTransformValue(&out.Items[i])).To(BeIdenticalTo("explicit")) + } + }) + + It("should apply default transformer to objects when none is specified", func(ctx SpecContext) { + By("getting the Kubernetes service") + svc := &unstructured.Unstructured{} + svc.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Service", + }) + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) + + By("verifying that the returned service was transformed") + Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) + }) + }) + + Context("with metadata-only objects", func() { + It("should apply transformers to explicitly specified GVKS", func(ctx SpecContext) { + By("listing pods") + out := metav1.PartialObjectMetadataList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + Expect(informerCache.List(ctx, &out)).To(Succeed()) + + By("verifying that the returned pods were transformed") + for i := 0; i < len(out.Items); i++ { + Expect(getTransformValue(&out.Items[i])).To(BeIdenticalTo("explicit")) + } + }) + It("should apply default transformer to objects when none is specified", func(ctx SpecContext) { + By("getting the Kubernetes service") + svc := &metav1.PartialObjectMetadata{} + svc.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Service", + }) + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) + + By("verifying that the returned service was transformed") + Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) + }) + }) }) -var _ = Describe("Informer Cache without DeepCopy", func() { - CacheTest(cache.New, cache.Options{UnsafeDisableDeepCopyByObject: cache.DisableDeepCopyByObject{cache.ObjectAll{}: true}}) + +var _ = Describe("Cache with selectors", func() { + defer GinkgoRecover() + var ( + informerCache cache.Cache + informerCacheCancel context.CancelFunc + ) + + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo + Expect(cfg).NotTo(BeNil()) + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceTwo, cl) + Expect(err).NotTo(HaveOccurred()) + for idx, namespace := range []string{testNamespaceOne, testNamespaceTwo} { + _ = createSA(ctx, "test-sa-"+strconv.Itoa(idx), namespace, cl) + _ = createSvc(ctx, "test-svc-"+strconv.Itoa(idx), namespace, cl) + } + + opts := cache.Options{ + DefaultFieldSelector: fields.OneTermEqualSelector("metadata.namespace", testNamespaceTwo), + ByObject: map[client.Object]cache.ByObject{ + &corev1.ServiceAccount{}: { + Field: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne), + }, + }, + } + + By("creating the informer cache") + informerCache, err = cache.New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + }) + + AfterEach(func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + for idx, namespace := range []string{testNamespaceOne, testNamespaceTwo} { + err = cl.Delete(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "test-sa-" + strconv.Itoa(idx)}}) + Expect(err).NotTo(HaveOccurred()) + err = cl.Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "test-svc-" + strconv.Itoa(idx)}}) + Expect(err).NotTo(HaveOccurred()) + } + informerCacheCancel() + }) + + It("Should list serviceaccounts and find exactly one in namespace "+testNamespaceOne, func(ctx SpecContext) { + var sas corev1.ServiceAccountList + err := informerCache.List(ctx, &sas) + Expect(err).NotTo(HaveOccurred()) + Expect(sas.Items).To(HaveLen(1)) + Expect(sas.Items[0].Namespace).To(Equal(testNamespaceOne)) + }) + + It("Should list services and find exactly one in namespace "+testNamespaceTwo, func(ctx SpecContext) { + var svcs corev1.ServiceList + err := informerCache.List(ctx, &svcs) + Expect(err).NotTo(HaveOccurred()) + Expect(svcs.Items).To(HaveLen(1)) + Expect(svcs.Items[0].Namespace).To(Equal(testNamespaceTwo)) + }) }) +func CacheTestReaderFailOnMissingInformer(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error), opts cache.Options) { + Describe("Cache test with ReaderFailOnMissingInformer = true", func() { + var ( + informerCache cache.Cache + informerCacheCancel context.CancelFunc + errNotCached *cache.ErrResourceNotCached + ) + + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo + Expect(cfg).NotTo(BeNil()) + By("creating the informer cache") + var err error + informerCache, err = createCacheFunc(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) + }) + + AfterEach(func() { + informerCacheCancel() + }) + + Describe("as a Reader", func() { + Context("with structured objects", func() { + It("should not be able to list objects that haven't been watched previously", func(ctx SpecContext) { + By("listing all services in the cluster") + listObj := &corev1.ServiceList{} + Expect(errors.As(informerCache.List(ctx, listObj), &errNotCached)).To(BeTrue()) + }) + + It("should not be able to get objects that haven't been watched previously", func(ctx SpecContext) { + By("getting the Kubernetes service") + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(errors.As(informerCache.Get(ctx, svcKey, svc), &errNotCached)).To(BeTrue()) + }) + + It("should be able to list objects that are configured to be watched", func(ctx SpecContext) { + By("indicating that we need to watch services") + _, err := informerCache.GetInformer(ctx, &corev1.Service{}) + Expect(err).ToNot(HaveOccurred()) + + By("listing all services in the cluster") + svcList := &corev1.ServiceList{} + Expect(informerCache.List(ctx, svcList)).To(Succeed()) + + By("verifying that the returned service looks reasonable") + Expect(svcList.Items).To(HaveLen(1)) + Expect(svcList.Items[0].Name).To(Equal("kubernetes")) + Expect(svcList.Items[0].Namespace).To(Equal("default")) + }) + + It("should be able to get objects that are configured to be watched", func(ctx SpecContext) { + By("indicating that we need to watch services") + _, err := informerCache.GetInformer(ctx, &corev1.Service{}) + Expect(err).ToNot(HaveOccurred()) + + By("getting the Kubernetes service") + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) + + By("verifying that the returned service looks reasonable") + Expect(svc.Name).To(Equal("kubernetes")) + Expect(svc.Namespace).To(Equal("default")) + }) + }) + }) + }) +} + +func NonBlockingGetTest(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error), opts cache.Options) { + Describe("non-blocking get test", func() { + var ( + informerCache cache.Cache + informerCacheCancel context.CancelFunc + ) + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo + Expect(cfg).NotTo(BeNil()) + + By("creating expected namespaces") + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + err = ensureNode(ctx, testNodeOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceTwo, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceThree, cl) + Expect(err).NotTo(HaveOccurred()) + + By("creating the informer cache") + opts.NewInformer = func(_ kcache.ListerWatcher, _ runtime.Object, _ time.Duration, _ kcache.Indexers) kcache.SharedIndexInformer { + return &controllertest.FakeInformer{Synced: false} + } + informerCache, err = createCacheFunc(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) + }) + + AfterEach(func() { + By("cleaning up created pods") + informerCacheCancel() + }) + + Describe("as an Informer", func() { + It("should be able to get informer for the object without blocking", func(specCtx SpecContext) { + By("getting a shared index informer for a pod") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "informer-obj", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + } + + ctx, cancel := context.WithTimeout(specCtx, 5*time.Second) + defer cancel() + sii, err := informerCache.GetInformer(ctx, pod, cache.BlockUntilSynced(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(sii).NotTo(BeNil()) + Expect(sii.HasSynced()).To(BeFalse()) + }) + }) + }) +} + func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error), opts cache.Options) { Describe("Cache test", func() { var ( informerCache cache.Cache - informerCacheCtx context.Context informerCacheCancel context.CancelFunc knownPod1 client.Object knownPod2 client.Object @@ -108,28 +612,33 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca knownPod6 client.Object ) - BeforeEach(func() { - informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo Expect(cfg).NotTo(BeNil()) By("creating three pods") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = ensureNode(testNodeOne, cl) + err = ensureNode(ctx, testNodeOne, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceOne, cl) + err = ensureNode(ctx, testNodeTwo, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceTwo, cl) + err = ensureNamespace(ctx, testNamespaceOne, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceThree, cl) + err = ensureNamespace(ctx, testNamespaceTwo, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceThree, cl) Expect(err).NotTo(HaveOccurred()) // Includes restart policy since these objects are indexed on this field. - knownPod1 = createPod("test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) - knownPod2 = createPod("test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) - knownPod3 = createPodWithLabels("test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) - knownPod4 = createPodWithLabels("test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) - knownPod5 = createPod("test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) - knownPod6 = createPod("test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod1 = createPod(ctx, "test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) + knownPod2 = createPod(ctx, "test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod3 = createPodWithLabels(ctx, "test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) + knownPod4 = createPodWithLabels(ctx, "test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) + knownPod5 = createPod(ctx, "test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) + knownPod6 = createPod(ctx, "test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) podGVK := schema.GroupVersionKind{ Kind: "Pod", @@ -152,27 +661,27 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca defer GinkgoRecover() Expect(informerCache.Start(ctx)).To(Succeed()) }(informerCacheCtx) - Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { By("cleaning up created pods") - deletePod(knownPod1) - deletePod(knownPod2) - deletePod(knownPod3) - deletePod(knownPod4) - deletePod(knownPod5) - deletePod(knownPod6) + deletePod(ctx, knownPod1) + deletePod(ctx, knownPod2) + deletePod(ctx, knownPod3) + deletePod(ctx, knownPod4) + deletePod(ctx, knownPod5) + deletePod(ctx, knownPod6) informerCacheCancel() }) Describe("as a Reader", func() { Context("with structured objects", func() { - It("should be able to list objects that haven't been watched previously", func() { + It("should be able to list objects that haven't been watched previously", func(ctx SpecContext) { By("listing all services in the cluster") listObj := &corev1.ServiceList{} - Expect(informerCache.List(context.Background(), listObj)).To(Succeed()) + Expect(informerCache.List(ctx, listObj)).To(Succeed()) By("verifying that the returned list contains the Kubernetes service") // NB: kubernetes default service is automatically created in testenv. @@ -188,22 +697,22 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(hasKubeService).To(BeTrue()) }) - It("should be able to get objects that haven't been watched previously", func() { + It("should be able to get objects that haven't been watched previously", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service looks reasonable") Expect(svc.Name).To(Equal("kubernetes")) Expect(svc.Namespace).To(Equal("default")) }) - It("should support filtering by labels in a single namespace", func() { + It("should support filtering by labels in a single namespace", func(ctx SpecContext) { By("listing pods with a particular label") // NB: each pod has a "test-label": out := corev1.PodList{} - Expect(informerCache.List(context.Background(), &out, + Expect(informerCache.List(ctx, &out, client.InNamespace(testNamespaceTwo), client.MatchingLabels(map[string]string{"test-label": "test-pod-2"}))).To(Succeed()) @@ -214,16 +723,16 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.Labels["test-label"]).To(Equal("test-pod-2")) }) - It("should support filtering by labels from multiple namespaces", func() { + It("should support filtering by labels from multiple namespaces", func(ctx SpecContext) { By("creating another pod with the same label but different namespace") - anotherPod := createPod("test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) - defer deletePod(anotherPod) + anotherPod := createPod(ctx, "test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) + defer deletePod(ctx, anotherPod) By("listing pods with a particular label") // NB: each pod has a "test-label": out := corev1.PodList{} labels := map[string]string{"test-label": "test-pod-2"} - Expect(informerCache.List(context.Background(), &out, client.MatchingLabels(labels))).To(Succeed()) + Expect(informerCache.List(ctx, &out, client.MatchingLabels(labels))).To(Succeed()) By("verifying multiple pods with the same label in different namespaces are returned") Expect(out.Items).NotTo(BeEmpty()) @@ -234,10 +743,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) if !isPodDisableDeepCopy(opts) { - It("should be able to list objects with GVK populated", func() { + It("should be able to list objects with GVK populated", func(ctx SpecContext) { By("listing pods") out := &corev1.PodList{} - Expect(informerCache.List(context.Background(), out)).To(Succeed()) + Expect(informerCache.List(ctx, out)).To(Succeed()) By("verifying that the returned pods have GVK populated") Expect(out.Items).NotTo(BeEmpty()) @@ -248,10 +757,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should be able to list objects by namespace", func() { + It("should be able to list objects by namespace", func(ctx SpecContext) { By("listing pods in test-namespace-1") listObj := &corev1.PodList{} - Expect(informerCache.List(context.Background(), listObj, + Expect(informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying that the returned pods are in test-namespace-1") @@ -263,11 +772,11 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) if !isPodDisableDeepCopy(opts) { - It("should deep copy the object unless told otherwise", func() { + It("should deep copy the object unless told otherwise", func(ctx SpecContext) { By("retrieving a specific pod from the cache") out := &corev1.Pod{} podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out)).To(Succeed()) By("verifying the retrieved pod is equal to a known pod") Expect(out).To(Equal(knownPod2)) @@ -279,13 +788,13 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(out).NotTo(Equal(knownPod2)) }) } else { - It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func(ctx SpecContext) { By("getting a specific pod from the cache twice") podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} out1 := &corev1.Pod{} - Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out1)).To(Succeed()) out2 := &corev1.Pod{} - Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out2)).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") Expect(out1).To(Equal(out2)) @@ -293,12 +802,12 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("listing pods from the cache twice") outList1 := &corev1.PodList{} - Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) outList2 := &corev1.PodList{} - Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") - Expect(len(outList1.Items)).To(Equal(len(outList2.Items))) + Expect(outList1.Items).To(HaveLen(len(outList2.Items))) sort.SliceStable(outList1.Items, func(i, j int) bool { return outList1.Items[i].Name <= outList1.Items[j].Name }) sort.SliceStable(outList2.Items, func(i, j int) bool { return outList2.Items[i].Name <= outList2.Items[j].Name }) for i := range outList1.Items { @@ -310,60 +819,81 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should return an error if the object is not found", func() { + It("should return an error if the object is not found", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: testNamespaceOne, Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return an error if getting object in unwatched namespace", func() { + It("should return an error if getting object in unwatched namespace", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "unknown", Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) }) - It("should return an error when context is cancelled", func() { + It("should return an error when context is cancelled", func(specCtx SpecContext) { By("cancelling the context") - informerCacheCancel() + ctx := cancelledCtx(specCtx) By("listing pods in test-namespace-1 with a cancelled context") listObj := &corev1.PodList{} - err := informerCache.List(informerCacheCtx, listObj, client.InNamespace(testNamespaceOne)) + err := informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne)) By("verifying that an error is returned") Expect(err).To(HaveOccurred()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) - It("should set the Limit option and limit number of objects to Limit when List is called", func() { + It("should set the Limit option and limit number of objects to Limit when List is called", func(ctx SpecContext) { opts := &client.ListOptions{Limit: int64(3)} By("verifying that only Limit (3) number of objects are retrieved from the cache") listObj := &corev1.PodList{} - Expect(informerCache.List(context.Background(), listObj, opts)).To(Succeed()) + Expect(informerCache.List(ctx, listObj, opts)).To(Succeed()) Expect(listObj.Items).Should(HaveLen(3)) }) - It("should return a limited result set matching the correct label", func() { + It("should return a limited result set matching the correct label", func(ctx SpecContext) { listObj := &corev1.PodList{} labelOpt := client.MatchingLabels(map[string]string{"common-label": "common"}) limitOpt := client.Limit(1) By("verifying that only Limit (1) number of objects are retrieved from the cache") - Expect(informerCache.List(context.Background(), listObj, labelOpt, limitOpt)).To(Succeed()) + Expect(informerCache.List(ctx, listObj, labelOpt, limitOpt)).To(Succeed()) Expect(listObj.Items).Should(HaveLen(1)) }) + + It("should return an error if pagination is used", func(ctx SpecContext) { + listObj := &corev1.PodList{} + By("verifying that the first list works and returns a sentinel continue") + err := informerCache.List(ctx, listObj) + Expect(err).ToNot(HaveOccurred()) + Expect(listObj.Continue).To(Equal("continue-not-supported")) + + By("verifying that an error is returned") + err = informerCache.List(ctx, listObj, client.Continue(listObj.Continue)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("continue list option is not supported by the cache")) + }) + + It("should return an error if the continue list options is set", func(ctx SpecContext) { + listObj := &corev1.PodList{} + continueOpt := client.Continue("token") + By("verifying that an error is returned") + err := informerCache.List(ctx, listObj, continueOpt) + Expect(err).To(HaveOccurred()) + }) }) Context("with unstructured objects", func() { - It("should be able to list objects that haven't been watched previously", func() { + It("should be able to list objects that haven't been watched previously", func(ctx SpecContext) { By("listing all services in the cluster") listObj := &unstructured.UnstructuredList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -371,7 +901,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "ServiceList", }) - err := informerCache.List(context.Background(), listObj) + err := informerCache.List(ctx, listObj) Expect(err).To(Succeed()) By("verifying that the returned list contains the Kubernetes service") @@ -387,7 +917,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } Expect(hasKubeService).To(BeTrue()) }) - It("should be able to get objects that haven't been watched previously", func() { + It("should be able to get objects that haven't been watched previously", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &unstructured.Unstructured{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -396,14 +926,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "Service", }) svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service looks reasonable") Expect(svc.GetName()).To(Equal("kubernetes")) Expect(svc.GetNamespace()).To(Equal("default")) }) - It("should support filtering by labels in a single namespace", func() { + It("should support filtering by labels in a single namespace", func(ctx SpecContext) { By("listing pods with a particular label") // NB: each pod has a "test-label": out := unstructured.UnstructuredList{} @@ -412,7 +942,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), &out, + err := informerCache.List(ctx, &out, client.InNamespace(testNamespaceTwo), client.MatchingLabels(map[string]string{"test-label": "test-pod-2"})) Expect(err).To(Succeed()) @@ -424,10 +954,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.GetLabels()["test-label"]).To(Equal("test-pod-2")) }) - It("should support filtering by labels from multiple namespaces", func() { + It("should support filtering by labels from multiple namespaces", func(ctx SpecContext) { By("creating another pod with the same label but different namespace") - anotherPod := createPod("test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) - defer deletePod(anotherPod) + anotherPod := createPod(ctx, "test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) + defer deletePod(ctx, anotherPod) By("listing pods with a particular label") // NB: each pod has a "test-label": @@ -438,7 +968,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "PodList", }) labels := map[string]string{"test-label": "test-pod-2"} - err := informerCache.List(context.Background(), &out, client.MatchingLabels(labels)) + err := informerCache.List(ctx, &out, client.MatchingLabels(labels)) Expect(err).To(Succeed()) By("verifying multiple pods with the same label in different namespaces are returned") @@ -449,7 +979,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to list objects by namespace", func() { + It("should be able to list objects by namespace", func(ctx SpecContext) { By("listing pods in test-namespace-1") listObj := &unstructured.UnstructuredList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -457,7 +987,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), listObj, client.InNamespace(testNamespaceOne)) + err := informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne)) Expect(err).To(Succeed()) By("verifying that the returned pods are in test-namespace-1") @@ -468,64 +998,94 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to restrict cache to a namespace", func() { - By("creating a namespaced cache") - namespacedCache, err := cache.New(cfg, cache.Options{Namespace: testNamespaceOne}) - Expect(err).NotTo(HaveOccurred()) + cacheRestrictSubTests := []struct { + nameSuffix string + cacheOpts cache.Options + }{ + { + nameSuffix: "by using the per-gvk setting", + cacheOpts: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Namespaces: map[string]cache.Config{ + testNamespaceOne: {}, + }, + }, + }, + }, + }, + { + nameSuffix: "by using the global DefaultNamespaces setting", + cacheOpts: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceOne: {}, + }, + }, + }, + } - By("running the cache and waiting for it to sync") - go func() { - defer GinkgoRecover() - Expect(namespacedCache.Start(informerCacheCtx)).To(Succeed()) - }() - Expect(namespacedCache.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + for _, tc := range cacheRestrictSubTests { + It("should be able to restrict cache to a namespace "+tc.nameSuffix, func(ctx SpecContext) { + By("creating a namespaced cache") + namespacedCache, err := cache.New(cfg, tc.cacheOpts) + Expect(err).NotTo(HaveOccurred()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(namespacedCache.Start(ctx)).To(Succeed()) + }() + Expect(namespacedCache.WaitForCacheSync(ctx)).To(BeTrue()) + + By("listing pods in all namespaces") + out := &unstructured.UnstructuredList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + for range 2 { + Expect(namespacedCache.List(ctx, out)).To(Succeed()) + + By("verifying the returned pod is from the watched namespace") + Expect(out.Items).NotTo(BeEmpty()) + Expect(out.Items).Should(HaveLen(2)) + for _, item := range out.Items { + Expect(item.GetNamespace()).To(Equal(testNamespaceOne)) + } + } + By("listing all nodes - should still be able to list a cluster-scoped resource") + nodeList := &unstructured.UnstructuredList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + Expect(namespacedCache.List(ctx, nodeList)).To(Succeed()) - By("listing pods in all namespaces") - out := &unstructured.UnstructuredList{} - out.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "PodList", - }) - Expect(namespacedCache.List(context.Background(), out)).To(Succeed()) + By("verifying the node list is not empty") + Expect(nodeList.Items).NotTo(BeEmpty()) - By("verifying the returned pod is from the watched namespace") - Expect(out.Items).NotTo(BeEmpty()) - Expect(out.Items).Should(HaveLen(2)) - for _, item := range out.Items { - Expect(item.GetNamespace()).To(Equal(testNamespaceOne)) - } - By("listing all nodes - should still be able to list a cluster-scoped resource") - nodeList := &unstructured.UnstructuredList{} - nodeList.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "NodeList", - }) - Expect(namespacedCache.List(context.Background(), nodeList)).To(Succeed()) + By("getting a node - should still be able to get a cluster-scoped resource") + node := &unstructured.Unstructured{} + node.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Node", + }) - By("verifying the node list is not empty") - Expect(nodeList.Items).NotTo(BeEmpty()) + By("verifying that getting the node works with an empty namespace") + key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} + Expect(namespacedCache.Get(ctx, key1, node)).To(Succeed()) - By("getting a node - should still be able to get a cluster-scoped resource") - node := &unstructured.Unstructured{} - node.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Node", + By("verifying that the namespace is ignored when getting a cluster-scoped resource") + key2 := client.ObjectKey{Namespace: "random", Name: testNodeOne} + Expect(namespacedCache.Get(ctx, key2, node)).To(Succeed()) }) - - By("verifying that getting the node works with an empty namespace") - key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key1, node)).To(Succeed()) - - By("verifying that the namespace is ignored when getting a cluster-scoped resource") - key2 := client.ObjectKey{Namespace: "random", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key2, node)).To(Succeed()) - }) + } if !isPodDisableDeepCopy(opts) { - It("should deep copy the object unless told otherwise", func() { + It("should deep copy the object unless told otherwise", func(ctx SpecContext) { By("retrieving a specific pod from the cache") out := &unstructured.Unstructured{} out.SetGroupVersionKind(schema.GroupVersionKind{ @@ -537,7 +1097,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(kscheme.Scheme.Convert(knownPod2, uKnownPod2, nil)).To(Succeed()) podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out)).To(Succeed()) By("verifying the retrieved pod is equal to a known pod") Expect(out).To(Equal(uKnownPod2)) @@ -550,15 +1110,15 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(out).NotTo(Equal(knownPod2)) }) } else { - It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func(ctx SpecContext) { By("getting a specific pod from the cache twice") podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} out1 := &unstructured.Unstructured{} out1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out1)).To(Succeed()) out2 := &unstructured.Unstructured{} out2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out2)).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") Expect(out1).To(Equal(out2)) @@ -567,13 +1127,13 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("listing pods from the cache twice") outList1 := &unstructured.UnstructuredList{} outList1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) outList2 := &unstructured.UnstructuredList{} outList2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") - Expect(len(outList1.Items)).To(Equal(len(outList2.Items))) + Expect(outList1.Items).To(HaveLen(len(outList2.Items))) sort.SliceStable(outList1.Items, func(i, j int) bool { return outList1.Items[i].GetName() <= outList1.Items[j].GetName() }) sort.SliceStable(outList2.Items, func(i, j int) bool { return outList2.Items[i].GetName() <= outList2.Items[j].GetName() }) for i := range outList1.Items { @@ -585,7 +1145,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should return an error if the object is not found", func() { + It("should return an error if the object is not found", func(ctx SpecContext) { By("getting a service that does not exists") svc := &unstructured.Unstructured{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -596,38 +1156,42 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca svcKey := client.ObjectKey{Namespace: testNamespaceOne, Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return an error if getting object in unwatched namespace", func() { + It("should return an error if getting object in unwatched namespace", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "unknown", Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) }) - It("test multinamespaced cache for cluster scoped resources", func() { + It("test multinamespaced cache for cluster scoped resources", func(ctx SpecContext) { By("creating a multinamespaced cache to watch specific namespaces") - multi := cache.MultiNamespacedCacheBuilder([]string{"default", testNamespaceOne}) - m, err := multi(cfg, cache.Options{}) + m, err := cache.New(cfg, cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + "default": {}, + testNamespaceOne: {}, + }, + }) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting it for sync") go func() { defer GinkgoRecover() - Expect(m.Start(informerCacheCtx)).To(Succeed()) + Expect(m.Start(ctx)).To(Succeed()) }() - Expect(m.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + Expect(m.WaitForCacheSync(ctx)).To(BeTrue()) By("should be able to fetch cluster scoped resource") node := &corev1.Node{} By("verifying that getting the node works with an empty namespace") key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} - Expect(m.Get(context.Background(), key1, node)).To(Succeed()) + Expect(m.Get(ctx, key1, node)).To(Succeed()) By("verifying if the cluster scoped resources are not duplicated") nodeList := &unstructured.UnstructuredList{} @@ -636,15 +1200,41 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "NodeList", }) - Expect(m.List(context.Background(), nodeList)).To(Succeed()) + Expect(m.List(ctx, nodeList)).To(Succeed()) By("verifying the node list is not empty") Expect(nodeList.Items).NotTo(BeEmpty()) - Expect(len(nodeList.Items)).To(BeEquivalentTo(1)) + Expect(len(nodeList.Items)).To(BeEquivalentTo(2)) + }) + + It("should return an error if pagination is used", func(ctx SpecContext) { + nodeList := &unstructured.UnstructuredList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + By("verifying that the first list works and returns a sentinel continue") + err := informerCache.List(ctx, nodeList) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeList.GetContinue()).To(Equal("continue-not-supported")) + + By("verifying that an error is returned") + err = informerCache.List(ctx, nodeList, client.Continue(nodeList.GetContinue())) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("continue list option is not supported by the cache")) + }) + + It("should return an error if the continue list options is set", func(ctx SpecContext) { + podList := &unstructured.Unstructured{} + continueOpt := client.Continue("token") + By("verifying that an error is returned") + err := informerCache.List(ctx, podList, continueOpt) + Expect(err).To(HaveOccurred()) }) }) Context("with metadata-only objects", func() { - It("should be able to list objects that haven't been watched previously", func() { + It("should be able to list objects that haven't been watched previously", func(ctx SpecContext) { By("listing all services in the cluster") listObj := &metav1.PartialObjectMetadataList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -652,7 +1242,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "ServiceList", }) - err := informerCache.List(context.Background(), listObj) + err := informerCache.List(ctx, listObj) Expect(err).To(Succeed()) By("verifying that the returned list contains the Kubernetes service") @@ -668,7 +1258,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } Expect(hasKubeService).To(BeTrue()) }) - It("should be able to get objects that haven't been watched previously", func() { + It("should be able to get objects that haven't been watched previously", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &metav1.PartialObjectMetadata{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -677,14 +1267,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "Service", }) svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service looks reasonable") Expect(svc.GetName()).To(Equal("kubernetes")) Expect(svc.GetNamespace()).To(Equal("default")) }) - It("should support filtering by labels in a single namespace", func() { + It("should support filtering by labels in a single namespace", func(ctx SpecContext) { By("listing pods with a particular label") // NB: each pod has a "test-label": out := metav1.PartialObjectMetadataList{} @@ -693,7 +1283,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), &out, + err := informerCache.List(ctx, &out, client.InNamespace(testNamespaceTwo), client.MatchingLabels(map[string]string{"test-label": "test-pod-2"})) Expect(err).To(Succeed()) @@ -705,10 +1295,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.GetLabels()["test-label"]).To(Equal("test-pod-2")) }) - It("should support filtering by labels from multiple namespaces", func() { + It("should support filtering by labels from multiple namespaces", func(ctx SpecContext) { By("creating another pod with the same label but different namespace") - anotherPod := createPod("test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) - defer deletePod(anotherPod) + anotherPod := createPod(ctx, "test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) + defer deletePod(ctx, anotherPod) By("listing pods with a particular label") // NB: each pod has a "test-label": @@ -719,7 +1309,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "PodList", }) labels := map[string]string{"test-label": "test-pod-2"} - err := informerCache.List(context.Background(), &out, client.MatchingLabels(labels)) + err := informerCache.List(ctx, &out, client.MatchingLabels(labels)) Expect(err).To(Succeed()) By("verifying multiple pods with the same label in different namespaces are returned") @@ -730,7 +1320,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to list objects by namespace", func() { + It("should be able to list objects by namespace", func(ctx SpecContext) { By("listing pods in test-namespace-1") listObj := &metav1.PartialObjectMetadataList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -738,7 +1328,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), listObj, client.InNamespace(testNamespaceOne)) + err := informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne)) Expect(err).To(Succeed()) By("verifying that the returned pods are in test-namespace-1") @@ -749,17 +1339,17 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to restrict cache to a namespace", func() { + It("should be able to restrict cache to a namespace", func(ctx SpecContext) { By("creating a namespaced cache") - namespacedCache, err := cache.New(cfg, cache.Options{Namespace: testNamespaceOne}) + namespacedCache, err := cache.New(cfg, cache.Options{DefaultNamespaces: map[string]cache.Config{testNamespaceOne: {}}}) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(namespacedCache.Start(informerCacheCtx)).To(Succeed()) + Expect(namespacedCache.Start(ctx)).To(Succeed()) }() - Expect(namespacedCache.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + Expect(namespacedCache.WaitForCacheSync(ctx)).To(BeTrue()) By("listing pods in all namespaces") out := &metav1.PartialObjectMetadataList{} @@ -768,7 +1358,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - Expect(namespacedCache.List(context.Background(), out)).To(Succeed()) + Expect(namespacedCache.List(ctx, out)).To(Succeed()) By("verifying the returned pod is from the watched namespace") Expect(out.Items).NotTo(BeEmpty()) @@ -783,7 +1373,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "NodeList", }) - Expect(namespacedCache.List(context.Background(), nodeList)).To(Succeed()) + Expect(namespacedCache.List(ctx, nodeList)).To(Succeed()) By("verifying the node list is not empty") Expect(nodeList.Items).NotTo(BeEmpty()) @@ -798,15 +1388,84 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("verifying that getting the node works with an empty namespace") key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key1, node)).To(Succeed()) + Expect(namespacedCache.Get(ctx, key1, node)).To(Succeed()) By("verifying that the namespace is ignored when getting a cluster-scoped resource") key2 := client.ObjectKey{Namespace: "random", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key2, node)).To(Succeed()) + Expect(namespacedCache.Get(ctx, key2, node)).To(Succeed()) + }) + + It("should be able to restrict cache to a namespace for namespaced object and to given selectors for non namespaced object", func(ctx SpecContext) { + By("creating a namespaced cache") + namespacedCache, err := cache.New(cfg, cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespaceOne: {}}, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Node{}: { + Label: labels.SelectorFromSet(labels.Set{"name": testNodeTwo}), + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(namespacedCache.Start(ctx)).To(Succeed()) + }() + Expect(namespacedCache.WaitForCacheSync(ctx)).To(BeTrue()) + + By("listing pods in all namespaces") + out := &metav1.PartialObjectMetadataList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + Expect(namespacedCache.List(ctx, out)).To(Succeed()) + + By("verifying the returned pod is from the watched namespace") + Expect(out.Items).NotTo(BeEmpty()) + Expect(out.Items).Should(HaveLen(2)) + for _, item := range out.Items { + Expect(item.Namespace).To(Equal(testNamespaceOne)) + } + By("listing all nodes - should still be able to list a cluster-scoped resource") + nodeList := &metav1.PartialObjectMetadataList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + Expect(namespacedCache.List(ctx, nodeList)).To(Succeed()) + + By("verifying the node list is not empty") + Expect(nodeList.Items).NotTo(BeEmpty()) + + By("getting a node - should still be able to get a cluster-scoped resource") + node := &metav1.PartialObjectMetadata{} + node.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Node", + }) + + By("verifying that getting the node works with an empty namespace") + key1 := client.ObjectKey{Namespace: "", Name: testNodeTwo} + Expect(namespacedCache.Get(ctx, key1, node)).To(Succeed()) + + By("verifying that the namespace is ignored when getting a cluster-scoped resource") + key2 := client.ObjectKey{Namespace: "random", Name: testNodeTwo} + Expect(namespacedCache.Get(ctx, key2, node)).To(Succeed()) + + By("verifying that an error is returned for node with not matching label") + key3 := client.ObjectKey{Namespace: "", Name: testNodeOne} + err = namespacedCache.Get(ctx, key3, node) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) if !isPodDisableDeepCopy(opts) { - It("should deep copy the object unless told otherwise", func() { + It("should deep copy the object unless told otherwise", func(ctx SpecContext) { By("retrieving a specific pod from the cache") out := &metav1.PartialObjectMetadata{} out.SetGroupVersionKind(schema.GroupVersionKind{ @@ -823,7 +1482,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out)).To(Succeed()) By("verifying the retrieved pod is equal to a known pod") Expect(out).To(Equal(uKnownPod2)) @@ -835,15 +1494,15 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(out).NotTo(Equal(knownPod2)) }) } else { - It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func(ctx SpecContext) { By("getting a specific pod from the cache twice") podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} out1 := &metav1.PartialObjectMetadata{} out1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out1)).To(Succeed()) out2 := &metav1.PartialObjectMetadata{} out2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out2)).To(Succeed()) By("verifying the pods have the same pointer addresses") By("verifying the pointer fields in pod have the same addresses") @@ -853,13 +1512,13 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("listing pods from the cache twice") outList1 := &metav1.PartialObjectMetadataList{} outList1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) outList2 := &metav1.PartialObjectMetadataList{} outList2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") - Expect(len(outList1.Items)).To(Equal(len(outList2.Items))) + Expect(outList1.Items).To(HaveLen(len(outList2.Items))) sort.SliceStable(outList1.Items, func(i, j int) bool { return outList1.Items[i].Name <= outList1.Items[j].Name }) sort.SliceStable(outList2.Items, func(i, j int) bool { return outList2.Items[i].Name <= outList2.Items[j].Name }) for i := range outList1.Items { @@ -871,7 +1530,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should return an error if the object is not found", func() { + It("should return an error if the object is not found", func(ctx SpecContext) { By("getting a service that does not exists") svc := &metav1.PartialObjectMetadata{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -882,50 +1541,57 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca svcKey := client.ObjectKey{Namespace: testNamespaceOne, Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return an error if getting object in unwatched namespace", func() { + It("should return an error if getting object in unwatched namespace", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "unknown", Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) + Expect(err).To(HaveOccurred()) + }) + + It("should return an error if pagination is used", func(ctx SpecContext) { + nodeList := &metav1.PartialObjectMetadataList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + By("verifying that the first list works and returns a sentinel continue") + err := informerCache.List(ctx, nodeList) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeList.GetContinue()).To(Equal("continue-not-supported")) + + By("verifying that an error is returned") + err = informerCache.List(ctx, nodeList, client.Continue(nodeList.GetContinue())) Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("continue list option is not supported by the cache")) }) }) type selectorsTestCase struct { - fieldSelectors map[string]string - labelSelectors map[string]string - expectedPods []string + options cache.Options + expectedPods []string } - DescribeTable(" and cache with selectors", func(tc selectorsTestCase) { + DescribeTable(" and cache with selectors", func(ctx SpecContext, tc selectorsTestCase) { By("creating the cache") - builder := cache.BuilderWithOptions( - cache.Options{ - SelectorsByObject: cache.SelectorsByObject{ - &corev1.Pod{}: { - Label: labels.Set(tc.labelSelectors).AsSelector(), - Field: fields.Set(tc.fieldSelectors).AsSelector(), - }, - }, - }, - ) - informer, err := builder(cfg, cache.Options{}) + informer, err := cache.New(cfg, tc.options) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("Checking with structured") obtainedStructuredPodList := corev1.PodList{} - Expect(informer.List(context.Background(), &obtainedStructuredPodList)).To(Succeed()) + Expect(informer.List(ctx, &obtainedStructuredPodList)).To(Succeed()) Expect(obtainedStructuredPodList.Items).Should(WithTransform(func(pods []corev1.Pod) []string { obtainedPodNames := []string{} for _, pod := range pods { @@ -933,6 +1599,9 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return obtainedPodNames }, ConsistOf(tc.expectedPods))) + for _, pod := range obtainedStructuredPodList.Items { + Expect(informer.Get(ctx, client.ObjectKeyFromObject(&pod), &pod)).To(Succeed()) + } By("Checking with unstructured") obtainedUnstructuredPodList := unstructured.UnstructuredList{} @@ -941,7 +1610,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err = informer.List(context.Background(), &obtainedUnstructuredPodList) + err = informer.List(ctx, &obtainedUnstructuredPodList) Expect(err).To(Succeed()) Expect(obtainedUnstructuredPodList.Items).Should(WithTransform(func(pods []unstructured.Unstructured) []string { obtainedPodNames := []string{} @@ -950,6 +1619,9 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return obtainedPodNames }, ConsistOf(tc.expectedPods))) + for _, pod := range obtainedUnstructuredPodList.Items { + Expect(informer.Get(ctx, client.ObjectKeyFromObject(&pod), &pod)).To(Succeed()) + } By("Checking with metadata") obtainedMetadataPodList := metav1.PartialObjectMetadataList{} @@ -958,7 +1630,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err = informer.List(context.Background(), &obtainedMetadataPodList) + err = informer.List(ctx, &obtainedMetadataPodList) Expect(err).To(Succeed()) Expect(obtainedMetadataPodList.Items).Should(WithTransform(func(pods []metav1.PartialObjectMetadata) []string { obtainedPodNames := []string{} @@ -967,46 +1639,300 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return obtainedPodNames }, ConsistOf(tc.expectedPods))) + for _, pod := range obtainedMetadataPodList.Items { + Expect(informer.Get(ctx, client.ObjectKeyFromObject(&pod), &pod)).To(Succeed()) + } }, Entry("when selectors are empty it has to inform about all the pods", selectorsTestCase{ - fieldSelectors: map[string]string{}, - labelSelectors: map[string]string{}, - expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, + expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, + }), + Entry("type-level field selector matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Field: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + })}, + }}, + expectedPods: []string{"test-pod-2"}, + }), + Entry("global field selector matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + }), + }, + expectedPods: []string{"test-pod-2"}, + }), + Entry("type-level field selectors matches multiple pods", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Field: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": testNamespaceTwo, + })}, + }}, + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-6"}, + }), + Entry("global field selectors matches multiple pods", selectorsTestCase{ + options: cache.Options{ + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": testNamespaceTwo, + }), + }, + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-6"}, + }), + Entry("type-level label selector matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-4", + })}, + }}, + expectedPods: []string{"test-pod-4"}, + }), + Entry("namespaces configured, type-level label selector matches everything, overrides global selector", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespaceOne: {}}, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.Everything()}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"does-not": "match-anything"}), + }, + expectedPods: []string{"test-pod-1", "test-pod-5"}, + }), + Entry("namespaces configured, global selector is used", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespaceTwo: {}}, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"common-label": "common"}), + }, + expectedPods: []string{"test-pod-3"}, + }), + Entry("global label selector matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-4", + }), + }, + expectedPods: []string{"test-pod-4"}, + }), + Entry("type-level label selector matches multiple pods", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.SelectorFromSet(map[string]string{ + "common-label": "common", + })}, + }}, + expectedPods: []string{"test-pod-3", "test-pod-4"}, + }), + Entry("global label selector matches multiple pods", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{ + "common-label": "common", + }), + }, + expectedPods: []string{"test-pod-3", "test-pod-4"}, + }), + Entry("type-level label and field selector, matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Label: labels.SelectorFromSet(map[string]string{"common-label": "common"}), + Field: fields.SelectorFromSet(map[string]string{"metadata.namespace": testNamespaceTwo}), + }, + }}, + expectedPods: []string{"test-pod-3"}, + }), + Entry("global label and field selector, matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"common-label": "common"}), + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{"metadata.namespace": testNamespaceTwo}), + }, + expectedPods: []string{"test-pod-3"}, + }), + Entry("type-level label selector does not match, no results", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.SelectorFromSet(map[string]string{ + "new-label": "new", + })}, + }}, + expectedPods: []string{}, }), - Entry("when field matches one pod it has to inform about it", selectorsTestCase{ - fieldSelectors: map[string]string{"metadata.name": "test-pod-2"}, - expectedPods: []string{"test-pod-2"}, + Entry("global label selector does not match, no results", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{ + "new-label": "new", + }), + }, + expectedPods: []string{}, + }), + Entry("type-level field selector does not match, no results", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Field: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": "new", + })}, + }}, + expectedPods: []string{}, + }), + Entry("global field selector does not match, no results", selectorsTestCase{ + options: cache.Options{ + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": "new", + }), + }, + expectedPods: []string{}, + }), + Entry("type-level field selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + }), + }, + }}, + }}, + expectedPods: []string{"test-pod-2"}, + }), + Entry("type-level field selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-doesn-exist", + }), + }, + }}, + }}, + expectedPods: []string{}, + }), + Entry("global field selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + }), + }, + }, + }, + expectedPods: []string{"test-pod-2"}, + }), + Entry("global field selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-doesn-exist", + }), + }, + }, + }, + expectedPods: []string{}, }), - Entry("when field matches multiple pods it has to inform about all of them", selectorsTestCase{ - fieldSelectors: map[string]string{"metadata.namespace": testNamespaceTwo}, - expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-6"}, + Entry("type-level label selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-2", + }), + }, + }}, + }}, + expectedPods: []string{"test-pod-2"}, + }), + Entry("type-level label selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-doesn-exist", + }), + }, + }}, + }}, + expectedPods: []string{}, + }), + Entry("global label selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-2", + }), + }, + }, + }, + expectedPods: []string{"test-pod-2"}, }), - Entry("when label matches one pod it has to inform about it", selectorsTestCase{ - labelSelectors: map[string]string{"test-label": "test-pod-4"}, - expectedPods: []string{"test-pod-4"}, + Entry("global label selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-doesn-exist", + }), + }, + }, + }, + expectedPods: []string{}, }), - Entry("when label matches multiple pods it has to inform about all of them", selectorsTestCase{ - labelSelectors: map[string]string{"common-label": "common"}, - expectedPods: []string{"test-pod-3", "test-pod-4"}, + Entry("Only NamespaceAll in DefaultNamespaces returns all pods", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + }, + }, + expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, }), - Entry("when label and field matches one pod it has to inform about about it", selectorsTestCase{ - labelSelectors: map[string]string{"common-label": "common"}, - fieldSelectors: map[string]string{"metadata.namespace": testNamespaceTwo}, - expectedPods: []string{"test-pod-3"}, + Entry("Only NamespaceAll in ByObject.Namespaces returns all pods", selectorsTestCase{ + options: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Namespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + }, + }, + }, + }, + expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, }), - Entry("when label does not match it does not has to inform", selectorsTestCase{ - labelSelectors: map[string]string{"new-label": "new"}, - expectedPods: []string{}, + Entry("NamespaceAll in DefaultNamespaces creates a cache for all Namespaces that are not in DefaultNamespaces", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + testNamespaceOne: { + // labels.Nothing when serialized matches everything, so we have to construct our own "match nothing" selector + LabelSelector: labels.SelectorFromSet(labels.Set{"no-present": "not-present"})}, + }, + }, + // All pods that are not in NamespaceOne + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-4", "test-pod-6"}, }), - Entry("when field does not match it does not has to inform", selectorsTestCase{ - fieldSelectors: map[string]string{"metadata.namespace": "new"}, - expectedPods: []string{}, + Entry("NamespaceAll in ByObject.Namespaces creates a cache for all Namespaces that are not in ByObject.Namespaces", selectorsTestCase{ + options: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Namespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + testNamespaceOne: { + // labels.Nothing when serialized matches everything, so we have to construct our own "match nothing" selector + LabelSelector: labels.SelectorFromSet(labels.Set{"no-present": "not-present"})}, + }, + }, + }, + }, + // All pods that are not in NamespaceOne + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-4", "test-pod-6"}, }), ) }) Describe("as an Informer", func() { + It("should error when starting the cache a second time", func(ctx SpecContext) { + err := informerCache.Start(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("informer already started")) + }) + Context("with structured objects", func() { - It("should be able to get informer for the object", func() { + It("should be able to get informer for the object", func(ctx SpecContext) { By("getting a shared index informer for a pod") pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -1022,7 +1948,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }, }, } - sii, err := informerCache.GetInformer(context.TODO(), pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1032,21 +1958,57 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) }) - It("should be able to get an informer by group/version/kind", func() { + It("should be able to stop and restart informers", func(ctx SpecContext) { + By("getting a shared index informer for a pod") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "informer-obj", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + } + sii, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii).NotTo(BeNil()) + Expect(sii.HasSynced()).To(BeTrue()) + + By("removing the existing informer") + Expect(informerCache.RemoveInformer(ctx, pod)).To(Succeed()) + Eventually(sii.IsStopped).WithTimeout(5 * time.Second).Should(BeTrue()) + + By("recreating the informer") + + sii2, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii2).NotTo(BeNil()) + Expect(sii2.HasSynced()).To(BeTrue()) + + By("validating the two informers are in different states") + Expect(sii.IsStopped()).To(BeTrue()) + Expect(sii2.IsStopped()).To(BeFalse()) + }) + It("should be able to get an informer by group/version/kind", func(ctx SpecContext) { By("getting an shared index informer for gvk = core/v1/pod") gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - sii, err := informerCache.GetInformerForKind(context.TODO(), gvk) + sii, err := informerCache.GetInformerForKind(ctx, gvk) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1056,7 +2018,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) @@ -1075,13 +2037,13 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }, }, } - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) }) - It("should be able to index an object field then retrieve objects by that field", func() { + It("should be able to index an object field then retrieve objects by that field", func(ctx SpecContext) { By("creating the cache") informer, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1091,18 +2053,18 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca indexFunc := func(obj client.Object) []string { return []string{string(obj.(*corev1.Pod).Spec.RestartPolicy)} } - Expect(informer.IndexField(context.TODO(), pod, "spec.restartPolicy", indexFunc)).To(Succeed()) + Expect(informer.IndexField(ctx, pod, "spec.restartPolicy", indexFunc)).To(Succeed()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("listing Pods with restartPolicyOnFailure") listObj := &corev1.PodList{} - Expect(informer.List(context.Background(), listObj, + Expect(informer.List(ctx, listObj, client.MatchingFields{"spec.restartPolicy": "OnFailure"})).To(Succeed()) By("verifying that the returned pods have correct restart policy") Expect(listObj.Items).NotTo(BeEmpty()) @@ -1111,9 +2073,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.Name).To(Equal("test-pod-3")) }) - It("should allow for get informer to be cancelled", func() { + It("should allow for get informer to be cancelled", func(specCtx SpecContext) { By("creating a context and cancelling it") - informerCacheCancel() + ctx, cancel := context.WithCancel(specCtx) + cancel() By("getting a shared index informer for a pod with a cancelled context") pod := &corev1.Pod{ @@ -1130,26 +2093,106 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }, }, } - sii, err := informerCache.GetInformer(informerCacheCtx, pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).To(HaveOccurred()) Expect(sii).To(BeNil()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) - It("should allow getting an informer by group/version/kind to be cancelled", func() { + It("should allow getting an informer by group/version/kind to be cancelled", func(specCtx SpecContext) { By("creating a context and cancelling it") - informerCacheCancel() + ctx, cancel := context.WithCancel(specCtx) + cancel() By("getting an shared index informer for gvk = core/v1/pod with a cancelled context") gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - sii, err := informerCache.GetInformerForKind(informerCacheCtx, gvk) + sii, err := informerCache.GetInformerForKind(ctx, gvk) Expect(err).To(HaveOccurred()) Expect(sii).To(BeNil()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) + + It("should be able not to change indexer values after indexing cluster-scope objects", func(ctx SpecContext) { + By("creating the cache") + informer, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + + By("indexing the Namespace objects with fixed values before starting") + ns := &corev1.Namespace{} + indexerValues := []string{"a", "b", "c"} + fieldName := "fixedValues" + indexFunc := func(obj client.Object) []string { + return indexerValues + } + Expect(informer.IndexField(ctx, ns, fieldName, indexFunc)).To(Succeed()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(informer.Start(ctx)).To(Succeed()) + }() + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) + + By("listing Namespaces with fixed indexer") + listObj := &corev1.NamespaceList{} + Expect(informer.List(ctx, listObj, + client.MatchingFields{fieldName: "a"})).To(Succeed()) + Expect(listObj.Items).NotTo(BeZero()) + + By("verifying the indexing does not change fixed returned values") + Expect(indexerValues).Should(HaveLen(3)) + Expect(indexerValues[0]).To(Equal("a")) + Expect(indexerValues[1]).To(Equal("b")) + Expect(indexerValues[2]).To(Equal("c")) + }) + + It("should be able to matching fields with multiple indexes", func(ctx SpecContext) { + By("creating the cache") + informer, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + + pod := &corev1.Pod{} + By("indexing pods with label before starting") + fieldName1 := "indexByLabel" + indexFunc1 := func(obj client.Object) []string { + return []string{obj.(*corev1.Pod).Labels["common-label"]} + } + Expect(informer.IndexField(ctx, pod, fieldName1, indexFunc1)).To(Succeed()) + By("indexing pods with restart policy before starting") + fieldName2 := "indexByPolicy" + indexFunc2 := func(obj client.Object) []string { + return []string{string(obj.(*corev1.Pod).Spec.RestartPolicy)} + } + Expect(informer.IndexField(ctx, pod, fieldName2, indexFunc2)).To(Succeed()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(informer.Start(ctx)).To(Succeed()) + }() + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) + + By("listing pods with label index") + listObj := &corev1.PodList{} + Expect(informer.List(ctx, listObj, + client.MatchingFields{fieldName1: "common"})).To(Succeed()) + Expect(listObj.Items).To(HaveLen(2)) + + By("listing pods with restart policy index") + listObj = &corev1.PodList{} + Expect(informer.List(ctx, listObj, + client.MatchingFields{fieldName2: string(corev1.RestartPolicyNever)})).To(Succeed()) + Expect(listObj.Items).To(HaveLen(3)) + + By("listing pods with both fixed indexers 1 and 2") + listObj = &corev1.PodList{} + Expect(informer.List(ctx, listObj, + client.MatchingFields{fieldName1: "common", fieldName2: string(corev1.RestartPolicyNever)})).To(Succeed()) + Expect(listObj.Items).To(HaveLen(1)) + }) }) Context("with unstructured objects", func() { - It("should be able to get informer for the object", func() { + It("should be able to get informer for the object", func(ctx SpecContext) { By("getting a shared index informer for a pod") pod := &unstructured.Unstructured{ @@ -1171,7 +2214,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "Pod", }) - sii, err := informerCache.GetInformer(context.TODO(), pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1181,19 +2224,60 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) - }, 3) + }) + + It("should be able to stop and restart informers", func(ctx SpecContext) { + By("getting a shared index informer for a pod") + pod := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "nginx", + "image": "nginx", + }, + }, + }, + }, + } + pod.SetName("informer-obj2") + pod.SetNamespace("default") + pod.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + }) + sii, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii).NotTo(BeNil()) + Expect(sii.HasSynced()).To(BeTrue()) - It("should be able to index an object field then retrieve objects by that field", func() { + By("removing the existing informer") + Expect(informerCache.RemoveInformer(ctx, pod)).To(Succeed()) + Eventually(sii.IsStopped).WithTimeout(5 * time.Second).Should(BeTrue()) + + By("recreating the informer") + sii2, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii2).NotTo(BeNil()) + Expect(sii2.HasSynced()).To(BeTrue()) + + By("validating the two informers are in different states") + Expect(sii.IsStopped()).To(BeTrue()) + Expect(sii2.IsStopped()).To(BeFalse()) + }) + + It("should be able to index an object field then retrieve objects by that field", func(ctx SpecContext) { By("creating the cache") informer, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1216,14 +2300,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return []string{fmt.Sprintf("%v", m["restartPolicy"])} } - Expect(informer.IndexField(context.TODO(), pod, "spec.restartPolicy", indexFunc)).To(Succeed()) + Expect(informer.IndexField(ctx, pod, "spec.restartPolicy", indexFunc)).To(Succeed()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("listing Pods with restartPolicyOnFailure") listObj := &unstructured.UnstructuredList{} @@ -1232,7 +2316,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err = informer.List(context.Background(), listObj, + err = informer.List(ctx, listObj, client.MatchingFields{"spec.restartPolicy": "OnFailure"}) Expect(err).To(Succeed()) @@ -1241,11 +2325,11 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(listObj.Items).Should(HaveLen(1)) actual := listObj.Items[0] Expect(actual.GetName()).To(Equal("test-pod-3")) - }, 3) + }) - It("should allow for get informer to be cancelled", func() { + It("should allow for get informer to be cancelled", func(specCtx SpecContext) { By("cancelling the context") - informerCacheCancel() + ctx := cancelledCtx(specCtx) By("getting a shared index informer for a pod with a cancelled context") pod := &unstructured.Unstructured{} @@ -1256,14 +2340,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "Pod", }) - sii, err := informerCache.GetInformer(informerCacheCtx, pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).To(HaveOccurred()) Expect(sii).To(BeNil()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) }) Context("with metadata-only objects", func() { - It("should be able to get informer for the object", func() { + It("should be able to get informer for the object", func(ctx SpecContext) { By("getting a shared index informer for a pod") pod := &corev1.Pod{ @@ -1289,7 +2373,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "Pod", }) - sii, err := informerCache.GetInformer(context.TODO(), podMeta) + sii, err := informerCache.GetInformer(ctx, podMeta) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1299,21 +2383,21 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) // re-copy the result in so that we can match on it properly pod.ObjectMeta.DeepCopyInto(&podMeta.ObjectMeta) By("verifying the object's metadata is received on the channel") Eventually(out).Should(Receive(Equal(podMeta))) - }, 3) + }) - It("should be able to index an object field then retrieve objects by that field", func() { + It("should be able to index an object field then retrieve objects by that field", func(ctx SpecContext) { By("creating the cache") informer, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1329,14 +2413,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca metadata := obj.(*metav1.PartialObjectMetadata) return []string{metadata.Labels["test-label"]} } - Expect(informer.IndexField(context.TODO(), pod, "metadata.labels.test-label", indexFunc)).To(Succeed()) + Expect(informer.IndexField(ctx, pod, "metadata.labels.test-label", indexFunc)).To(Succeed()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("listing Pods with restartPolicyOnFailure") listObj := &metav1.PartialObjectMetadataList{} @@ -1346,7 +2430,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "PodList", } listObj.SetGroupVersionKind(gvk) - err = informer.List(context.Background(), listObj, + err = informer.List(ctx, listObj, client.MatchingFields{"metadata.labels.test-label": "test-pod-3"}) Expect(err).To(Succeed()) @@ -1365,12 +2449,11 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "Pod", })) - }, 3) + }) - It("should allow for get informer to be cancelled", func() { + It("should allow for get informer to be cancelled", func(specContext SpecContext) { By("creating a context and cancelling it") - ctx, cancel := context.WithCancel(context.Background()) - cancel() + ctx := cancelledCtx(specContext) By("getting a shared index informer for a pod with a cancelled context") pod := &metav1.PartialObjectMetadata{} @@ -1388,11 +2471,69 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) }) }) + Context("using UnsafeDisableDeepCopy", func() { + Describe("with ListOptions", func() { + It("should be able to change object in informer cache", func(ctx SpecContext) { + By("listing pods") + out := corev1.PodList{} + Expect(informerCache.List(ctx, &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + for _, item := range out.Items { + if strings.Compare(item.Name, "test-pod-3") == 0 { // test-pod-3 has labels + item.Labels["UnsafeDisableDeepCopy"] = "true" + break + } + } + + By("verifying that the returned pods were changed") + out2 := corev1.PodList{} + Expect(informerCache.List(ctx, &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + for _, item := range out2.Items { + if strings.Compare(item.Name, "test-pod-3") == 0 { + Expect(item.Labels["UnsafeDisableDeepCopy"]).To(Equal("true")) + break + } + } + }) + }) + Describe("with GetOptions", func() { + It("should be able to change object in informer cache", func(ctx SpecContext) { + out := corev1.Pod{} + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + Expect(informerCache.Get(ctx, podKey, &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + + out.Labels["UnsafeDisableDeepCopy"] = "true" + + By("verifying that the returned pod was changed") + out2 := corev1.Pod{} + Expect(informerCache.Get(ctx, podKey, &out2, client.UnsafeDisableDeepCopy)).To(Succeed()) + Expect(out2.Labels["UnsafeDisableDeepCopy"]).To(Equal("true")) + }) + }) + }) }) } +var _ = Describe("TransformStripManagedFields", func() { + It("should strip managed fields from an object", func() { + obj := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "foo", + }}, + }} + transformed, err := cache.TransformStripManagedFields()(obj) + Expect(err).NotTo(HaveOccurred()) + Expect(transformed).To(Equal(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{}})) + }) + + It("should not trip over an unexpected object", func() { + transformed, err := cache.TransformStripManagedFields()("foo") + Expect(err).NotTo(HaveOccurred()) + Expect(transformed).To(Equal("foo")) + }) +}) + // ensureNamespace installs namespace of a given name if not exists. -func ensureNamespace(namespace string, client client.Client) error { +func ensureNamespace(ctx context.Context, namespace string, client client.Client) error { ns := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, @@ -1402,43 +2543,48 @@ func ensureNamespace(namespace string, client client.Client) error { APIVersion: "v1", }, } - err := client.Create(context.TODO(), &ns) + err := client.Create(ctx, &ns) if apierrors.IsAlreadyExists(err) { return nil } return err } -func ensureNode(name string, client client.Client) error { +func ensureNode(ctx context.Context, name string, client client.Client) error { node := corev1.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + Labels: map[string]string{"name": name}, }, TypeMeta: metav1.TypeMeta{ Kind: "Node", APIVersion: "v1", }, } - err := client.Create(context.TODO(), &node) + err := client.Create(ctx, &node) if apierrors.IsAlreadyExists(err) { return nil } return err } -//nolint:interfacer func isKubeService(svc metav1.Object) bool { // grumble grumble linters grumble grumble return svc.GetNamespace() == "default" && svc.GetName() == "kubernetes" } func isPodDisableDeepCopy(opts cache.Options) bool { - if d, ok := opts.UnsafeDisableDeepCopyByObject[&corev1.Pod{}]; ok { - return d - } else if d, ok = opts.UnsafeDisableDeepCopyByObject[cache.ObjectAll{}]; ok { - return d - } else if d, ok = opts.UnsafeDisableDeepCopyByObject[&cache.ObjectAll{}]; ok { - return d + if opts.ByObject[&corev1.Pod{}].UnsafeDisableDeepCopy != nil { + return *opts.ByObject[&corev1.Pod{}].UnsafeDisableDeepCopy + } + if opts.DefaultUnsafeDisableDeepCopy != nil { + return *opts.DefaultUnsafeDisableDeepCopy } return false } + +func cancelledCtx(ctx context.Context) context.Context { + cancelCtx, cancel := context.WithCancel(ctx) + cancel() + return cancelCtx +} diff --git a/pkg/cache/defaulting_test.go b/pkg/cache/defaulting_test.go new file mode 100644 index 0000000000..d9d0dcceb3 --- /dev/null +++ b/pkg/cache/defaulting_test.go @@ -0,0 +1,498 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "reflect" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + fuzz "github.com/google/gofuzz" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestDefaultOpts(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{} + + compare := func(a, b any) string { + return cmp.Diff(a, b, + cmpopts.IgnoreUnexported(Options{}), + cmpopts.IgnoreFields(Options{}, "HTTPClient", "Scheme", "Mapper", "SyncPeriod"), + cmp.Comparer(func(a, b fields.Selector) bool { + if (a != nil) != (b != nil) { + return false + } + if a == nil { + return true + } + return a.String() == b.String() + }), + ) + } + testCases := []struct { + name string + in Options + + verification func(Options) string + }{ + { + name: "ByObject.Namespaces gets defaulted from ByObject", + in: Options{ + ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "default": {}, + }, + Label: labels.SelectorFromSet(map[string]string{"from": "by-object"}), + }}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "by-object"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces gets defaulted from DefaultNamespaces", + in: Options{ + ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "default": {}, + }, + }}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces gets defaulted from DefaultLabelSelector", + in: Options{ + ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "default": {}, + }, + }}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces gets defaulted from DefaultNamespaces", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces doesn't get defaulted when its empty", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {Namespaces: map[string]Config{}}}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + }, + + verification: func(o Options) string { + expected := map[string]Config{} + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Labels gets defaulted from DefautLabelSelector", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}) + return cmp.Diff(expected, o.ByObject[pod].Label) + }, + }, + { + name: "ByObject.Labels doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {Label: labels.SelectorFromSet(map[string]string{"from": "by-object"})}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := labels.SelectorFromSet(map[string]string{"from": "by-object"}) + return cmp.Diff(expected, o.ByObject[pod].Label) + }, + }, + { + name: "ByObject.Fields gets defaulted from DefaultFieldSelector", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{"from": "default-field-selector"}), + }, + + verification: func(o Options) string { + expected := fields.SelectorFromSet(map[string]string{"from": "default-field-selector"}) + return cmp.Diff(expected, o.ByObject[pod].Field, cmp.Exporter(func(reflect.Type) bool { return true })) + }, + }, + { + name: "ByObject.Fields doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {Field: fields.SelectorFromSet(map[string]string{"from": "by-object"})}}, + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{"from": "default-field-selector"}), + }, + + verification: func(o Options) string { + expected := fields.SelectorFromSet(map[string]string{"from": "by-object"}) + return cmp.Diff(expected, o.ByObject[pod].Field, cmp.Exporter(func(reflect.Type) bool { return true })) + }, + }, + { + name: "ByObject.UnsafeDisableDeepCopy gets defaulted from DefaultUnsafeDisableDeepCopy", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultUnsafeDisableDeepCopy: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(true) + return cmp.Diff(expected, o.ByObject[pod].UnsafeDisableDeepCopy) + }, + }, + { + name: "ByObject.UnsafeDisableDeepCopy doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {UnsafeDisableDeepCopy: ptr.To(false)}}, + DefaultUnsafeDisableDeepCopy: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(false) + return cmp.Diff(expected, o.ByObject[pod].UnsafeDisableDeepCopy) + }, + }, + { + name: "ByObject.EnableWatchBookmarks gets defaulted from DefaultEnableWatchBookmarks", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultEnableWatchBookmarks: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(true) + return cmp.Diff(expected, o.ByObject[pod].EnableWatchBookmarks) + }, + }, + { + name: "ByObject.EnableWatchBookmarks doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {EnableWatchBookmarks: ptr.To(false)}}, + DefaultEnableWatchBookmarks: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(false) + return cmp.Diff(expected, o.ByObject[pod].EnableWatchBookmarks) + }, + }, + { + name: "DefaultNamespace label selector gets defaulted from DefaultLabelSelector", + in: Options{ + DefaultNamespaces: map[string]Config{"default": {}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"})}, + } + return cmp.Diff(expected, o.DefaultNamespaces) + }, + }, + { + name: "ByObject.Namespaces get selector from DefaultNamespaces before DefaultSelector", + in: Options{ + ByObject: map[client.Object]ByObject{ + pod: {Namespaces: map[string]Config{"default": {}}}, + }, + DefaultNamespaces: map[string]Config{"default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "namespace"})}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default"}), + }, + + verification: func(o Options) string { + expected := Options{ + ByObject: map[client.Object]ByObject{ + pod: {Namespaces: map[string]Config{"default": { + LabelSelector: labels.SelectorFromSet(map[string]string{"from": "namespace"}), + }}}, + }, + DefaultNamespaces: map[string]Config{"default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "namespace"})}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default"}), + } + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in DefaultNamespaces with custom selection logic", + in: Options{DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {}, + }}, + + verification: func(o Options) string { + expected := Options{ + DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie("metadata.namespace!=kube-public,metadata.namespace!=kube-system")}, + }, + } + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in DefaultNamespaces with custom selection logic and namespace default has its own field selector", + in: Options{DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie("spec.nodeName=foo")}, + }}, + + verification: func(o Options) string { + expected := Options{ + DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie( + "metadata.namespace!=kube-public,metadata.namespace!=kube-system,spec.nodeName=foo", + )}, + }, + } + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in ByObject.Namespaces with custom selection logic", + in: Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {}, + }, + }}}, + + verification: func(o Options) string { + expected := Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie( + "metadata.namespace!=kube-public,metadata.namespace!=kube-system", + )}, + }, + }}} + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in ByObject.Namespaces with custom selection logic and namespace default has its own field selector", + in: Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie("spec.nodeName=foo")}, + }, + }}}, + + verification: func(o Options) string { + expected := Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie( + "metadata.namespace!=kube-public,metadata.namespace!=kube-system,spec.nodeName=foo", + )}, + }, + }}} + + return compare(expected, o) + }, + }, + { + name: "DefaultNamespace label selector doesn't get defaulted when set", + in: Options{ + DefaultNamespaces: map[string]Config{"default": {LabelSelector: labels.Everything()}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.Everything()}, + } + return cmp.Diff(expected, o.DefaultNamespaces) + }, + }, + { + name: "Defaulted namespaces in ByObject contain ByObject's selector", + in: Options{ + ByObject: map[client.Object]ByObject{ + pod: {Label: labels.SelectorFromSet(map[string]string{"from": "pod"})}, + }, + DefaultNamespaces: map[string]Config{"default": {}}, + }, + verification: func(o Options) string { + expected := Options{ + ByObject: map[client.Object]ByObject{ + pod: { + Label: labels.SelectorFromSet(map[string]string{"from": "pod"}), + Namespaces: map[string]Config{"default": { + LabelSelector: labels.SelectorFromSet(map[string]string{"from": "pod"}), + }}, + }, + }, + + DefaultNamespaces: map[string]Config{"default": {}}, + } + return compare(expected, o) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.in.Mapper = &fakeRESTMapper{} + + defaulted, err := defaultOpts(&rest.Config{}, tc.in) + if err != nil { + t.Fatal(err) + } + + if diff := tc.verification(defaulted); diff != "" { + t.Errorf("expected config differs from actual: %s", diff) + } + }) + } +} + +func TestDefaultOptsRace(t *testing.T) { + opts := Options{ + Mapper: &fakeRESTMapper{}, + ByObject: map[client.Object]ByObject{ + &corev1.Pod{}: { + Label: labels.SelectorFromSet(map[string]string{"from": "pod"}), + Namespaces: map[string]Config{"default": { + LabelSelector: labels.SelectorFromSet(map[string]string{"from": "pod"}), + }}, + }, + }, + DefaultNamespaces: map[string]Config{"default": {}}, + } + + // Start go routines which re-use the above options struct. + wg := sync.WaitGroup{} + for range 2 { + wg.Add(1) + go func() { + _, _ = defaultOpts(&rest.Config{}, opts) + wg.Done() + }() + } + + // Wait for the go routines to finish. + wg.Wait() +} + +type fakeRESTMapper struct { + meta.RESTMapper +} + +func (f *fakeRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + return &meta.RESTMapping{Scope: meta.RESTScopeNamespace}, nil +} + +func TestDefaultConfigConsidersAllFields(t *testing.T) { + t.Parallel() + seed := time.Now().UnixNano() + t.Logf("Seed is %d", seed) + f := fuzz.NewWithSeed(seed).Funcs( + func(ls *labels.Selector, _ fuzz.Continue) { + *ls = labels.SelectorFromSet(map[string]string{"foo": "bar"}) + }, + func(fs *fields.Selector, _ fuzz.Continue) { + *fs = fields.SelectorFromSet(map[string]string{"foo": "bar"}) + }, + func(tf *cache.TransformFunc, _ fuzz.Continue) { + // never default this, as functions can not be compared so we fail down the line + }, + ) + + for i := 0; i < 100; i++ { + fuzzed := Config{} + f.Fuzz(&fuzzed) + + defaulted := defaultConfig(Config{}, fuzzed) + + if diff := cmp.Diff(fuzzed, defaulted, cmp.Exporter(func(reflect.Type) bool { return true })); diff != "" { + t.Errorf("Defaulted config doesn't match fuzzed one: %s", diff) + } + } +} diff --git a/pkg/cache/delegating_by_gvk_cache.go b/pkg/cache/delegating_by_gvk_cache.go new file mode 100644 index 0000000000..46bd243c66 --- /dev/null +++ b/pkg/cache/delegating_by_gvk_cache.go @@ -0,0 +1,136 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "context" + "maps" + "slices" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +// delegatingByGVKCache delegates to a type-specific cache if present +// and uses the defaultCache otherwise. +type delegatingByGVKCache struct { + scheme *runtime.Scheme + caches map[schema.GroupVersionKind]Cache + defaultCache Cache +} + +func (dbt *delegatingByGVKCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return err + } + return cache.Get(ctx, key, obj, opts...) +} + +func (dbt *delegatingByGVKCache) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + cache, err := dbt.cacheForObject(list) + if err != nil { + return err + } + return cache.List(ctx, list, opts...) +} + +func (dbt *delegatingByGVKCache) RemoveInformer(ctx context.Context, obj client.Object) error { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return err + } + return cache.RemoveInformer(ctx, obj) +} + +func (dbt *delegatingByGVKCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return nil, err + } + return cache.GetInformer(ctx, obj, opts...) +} + +func (dbt *delegatingByGVKCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) { + return dbt.cacheForGVK(gvk).GetInformerForKind(ctx, gvk, opts...) +} + +func (dbt *delegatingByGVKCache) Start(ctx context.Context) error { + allCaches := slices.Collect(maps.Values(dbt.caches)) + allCaches = append(allCaches, dbt.defaultCache) + + wg := &sync.WaitGroup{} + errs := make(chan error) + for idx := range allCaches { + cache := allCaches[idx] + wg.Add(1) + go func() { + defer wg.Done() + if err := cache.Start(ctx); err != nil { + errs <- err + } + }() + } + + select { + case err := <-errs: + return err + case <-ctx.Done(): + wg.Wait() + return nil + } +} + +func (dbt *delegatingByGVKCache) WaitForCacheSync(ctx context.Context) bool { + synced := true + for _, cache := range append(slices.Collect(maps.Values(dbt.caches)), dbt.defaultCache) { + if !cache.WaitForCacheSync(ctx) { + synced = false + } + } + + return synced +} + +func (dbt *delegatingByGVKCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return err + } + return cache.IndexField(ctx, obj, field, extractValue) +} + +func (dbt *delegatingByGVKCache) cacheForObject(o runtime.Object) (Cache, error) { + gvk, err := apiutil.GVKForObject(o, dbt.scheme) + if err != nil { + return nil, err + } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + return dbt.cacheForGVK(gvk), nil +} + +func (dbt *delegatingByGVKCache) cacheForGVK(gvk schema.GroupVersionKind) Cache { + if specific, hasSpecific := dbt.caches[gvk]; hasSpecific { + return specific + } + + return dbt.defaultCache +} diff --git a/pkg/cache/informer_cache.go b/pkg/cache/informer_cache.go index 90647c8e33..091667b7fa 100644 --- a/pkg/cache/informer_cache.go +++ b/pkg/cache/informer_cache.go @@ -19,14 +19,15 @@ package cache import ( "context" "fmt" - "reflect" "strings" apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache/internal" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -45,19 +46,38 @@ func (*ErrCacheNotStarted) Error() string { return "the cache is not started, can not read objects" } -// informerCache is a Kubernetes Object cache populated from InformersMap. informerCache wraps an InformersMap. +var _ error = (*ErrCacheNotStarted)(nil) + +// ErrResourceNotCached indicates that the resource type +// the client asked the cache for is not cached, i.e. the +// corresponding informer does not exist yet. +type ErrResourceNotCached struct { + GVK schema.GroupVersionKind +} + +// Error returns the error +func (r ErrResourceNotCached) Error() string { + return fmt.Sprintf("%s is not cached", r.GVK.String()) +} + +var _ error = (*ErrResourceNotCached)(nil) + +// informerCache is a Kubernetes Object cache populated from internal.Informers. +// informerCache wraps internal.Informers. type informerCache struct { - *internal.InformersMap + scheme *runtime.Scheme + *internal.Informers + readerFailOnMissingInformer bool } // Get implements Reader. -func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out client.Object) error { - gvk, err := apiutil.GVKForObject(out, ip.Scheme) +func (ic *informerCache) Get(ctx context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error { + gvk, err := apiutil.GVKForObject(out, ic.scheme) if err != nil { return err } - started, cache, err := ip.InformersMap.Get(ctx, gvk, out) + started, cache, err := ic.getInformerForKind(ctx, gvk, out) if err != nil { return err } @@ -65,17 +85,17 @@ func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out clie if !started { return &ErrCacheNotStarted{} } - return cache.Reader.Get(ctx, key, out) + return cache.Reader.Get(ctx, key, out, opts...) } // List implements Reader. -func (ip *informerCache) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error { - gvk, cacheTypeObj, err := ip.objectTypeForListObject(out) +func (ic *informerCache) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error { + gvk, cacheTypeObj, err := ic.objectTypeForListObject(out) if err != nil { return err } - started, cache, err := ip.InformersMap.Get(ctx, *gvk, cacheTypeObj) + started, cache, err := ic.getInformerForKind(ctx, *gvk, cacheTypeObj) if err != nil { return err } @@ -90,94 +110,117 @@ func (ip *informerCache) List(ctx context.Context, out client.ObjectList, opts . // objectTypeForListObject tries to find the runtime.Object and associated GVK // for a single object corresponding to the passed-in list type. We need them // because they are used as cache map key. -func (ip *informerCache) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) { - gvk, err := apiutil.GVKForObject(list, ip.Scheme) +func (ic *informerCache) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) { + gvk, err := apiutil.GVKForObject(list, ic.scheme) if err != nil { return nil, nil, err } - if !strings.HasSuffix(gvk.Kind, "List") { - return nil, nil, fmt.Errorf("non-list type %T (kind %q) passed as output", list, gvk) - } - // we need the non-list GVK, so chop off the "List" from the end of the kind - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - _, isUnstructured := list.(*unstructured.UnstructuredList) - var cacheTypeObj runtime.Object - if isUnstructured { + // We need the non-list GVK, so chop off the "List" from the end of the kind. + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + + // Handle unstructured.UnstructuredList. + if _, isUnstructured := list.(runtime.Unstructured); isUnstructured { u := &unstructured.Unstructured{} u.SetGroupVersionKind(gvk) - cacheTypeObj = u - } else { - itemsPtr, err := apimeta.GetItemsPtr(list) - if err != nil { - return nil, nil, err - } - // http://knowyourmeme.com/memes/this-is-fine - elemType := reflect.Indirect(reflect.ValueOf(itemsPtr)).Type().Elem() - if elemType.Kind() != reflect.Ptr { - elemType = reflect.PtrTo(elemType) - } - - cacheTypeValue := reflect.Zero(elemType) - var ok bool - cacheTypeObj, ok = cacheTypeValue.Interface().(runtime.Object) - if !ok { - return nil, nil, fmt.Errorf("cannot get cache for %T, its element %T is not a runtime.Object", list, cacheTypeValue.Interface()) - } + return &gvk, u, nil + } + // Handle metav1.PartialObjectMetadataList. + if _, isPartialObjectMetadata := list.(*metav1.PartialObjectMetadataList); isPartialObjectMetadata { + pom := &metav1.PartialObjectMetadata{} + pom.SetGroupVersionKind(gvk) + return &gvk, pom, nil } + // Any other list type should have a corresponding non-list type registered + // in the scheme. Use that to create a new instance of the non-list type. + cacheTypeObj, err := ic.scheme.New(gvk) + if err != nil { + return nil, nil, err + } return &gvk, cacheTypeObj, nil } -// GetInformerForKind returns the informer for the GroupVersionKind. -func (ip *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) { +func applyGetOptions(opts ...InformerGetOption) *internal.GetOptions { + cfg := &InformerGetOptions{} + for _, opt := range opts { + opt(cfg) + } + return (*internal.GetOptions)(cfg) +} + +// GetInformerForKind returns the informer for the GroupVersionKind. If no informer exists, one will be started. +func (ic *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) { // Map the gvk to an object - obj, err := ip.Scheme.New(gvk) + obj, err := ic.scheme.New(gvk) if err != nil { return nil, err } - _, i, err := ip.InformersMap.Get(ctx, gvk, obj) + _, i, err := ic.Informers.Get(ctx, gvk, obj, applyGetOptions(opts...)) if err != nil { return nil, err } - return i.Informer, err + return i.Informer, nil } -// GetInformer returns the informer for the obj. -func (ip *informerCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) { - gvk, err := apiutil.GVKForObject(obj, ip.Scheme) +// GetInformer returns the informer for the obj. If no informer exists, one will be started. +func (ic *informerCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) { + gvk, err := apiutil.GVKForObject(obj, ic.scheme) if err != nil { return nil, err } - _, i, err := ip.InformersMap.Get(ctx, gvk, obj) + _, i, err := ic.Informers.Get(ctx, gvk, obj, applyGetOptions(opts...)) if err != nil { return nil, err } - return i.Informer, err + return i.Informer, nil +} + +func (ic *informerCache) getInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *internal.Cache, error) { + if ic.readerFailOnMissingInformer { + cache, started, ok := ic.Informers.Peek(gvk, obj) + if !ok { + return false, nil, &ErrResourceNotCached{GVK: gvk} + } + return started, cache, nil + } + + return ic.Informers.Get(ctx, gvk, obj, &internal.GetOptions{}) +} + +// RemoveInformer deactivates and removes the informer from the cache. +func (ic *informerCache) RemoveInformer(_ context.Context, obj client.Object) error { + gvk, err := apiutil.GVKForObject(obj, ic.scheme) + if err != nil { + return err + } + + ic.Informers.Remove(gvk, obj) + return nil } // NeedLeaderElection implements the LeaderElectionRunnable interface // to indicate that this can be started without requiring the leader lock. -func (ip *informerCache) NeedLeaderElection() bool { +func (ic *informerCache) NeedLeaderElection() bool { return false } -// IndexField adds an indexer to the underlying cache, using extraction function to get -// value(s) from the given field. This index can then be used by passing a field selector +// IndexField adds an indexer to the underlying informer, using extractValue function to get +// value(s) from the given field. This index can then be used by passing a field selector // to List. For one-to-one compatibility with "normal" field selectors, only return one value. -// The values may be anything. They will automatically be prefixed with the namespace of the -// given object, if present. The objects passed are guaranteed to be objects of the correct type. -func (ip *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { - informer, err := ip.GetInformer(ctx, obj) +// The values may be anything. They will automatically be prefixed with the namespace of the +// given object, if present. The objects passed are guaranteed to be objects of the correct type. +func (ic *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + informer, err := ic.GetInformer(ctx, obj) if err != nil { return err } return indexByField(informer, field, extractValue) } -func indexByField(indexer Informer, field string, extractor client.IndexerFunc) error { +func indexByField(informer Informer, field string, extractValue client.IndexerFunc) error { indexFunc := func(objRaw interface{}) ([]string, error) { // TODO(directxman12): check if this is the correct type? obj, isObj := objRaw.(client.Object) @@ -190,11 +233,11 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc) } ns := meta.GetNamespace() - rawVals := extractor(obj) + rawVals := extractValue(obj) var vals []string if ns == "" { - // if we're not doubling the keys for the namespaced case, just re-use what was returned to us - vals = rawVals + // if we're not doubling the keys for the namespaced case, just create a new slice with same length + vals = make([]string, len(rawVals)) } else { // if we need to add non-namespaced versions too, double the length vals = make([]string, len(rawVals)*2) @@ -213,5 +256,5 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc) return vals, nil } - return indexer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc}) + return informer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc}) } diff --git a/pkg/cache/informer_cache_test.go b/pkg/cache/informer_cache_test.go index 6a19eaa366..617e74c4e5 100644 --- a/pkg/cache/informer_cache_test.go +++ b/pkg/cache/informer_cache_test.go @@ -17,7 +17,7 @@ limitations under the License. package cache_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" @@ -31,7 +31,9 @@ var _ = Describe("informerCache", func() { It("should not require LeaderElection", func() { cfg := &rest.Config{} - mapper, err := apiutil.NewDynamicRESTMapper(cfg, apiutil.WithLazyDiscovery) + httpClient, err := rest.HTTPClientFor(cfg) + Expect(err).ToNot(HaveOccurred()) + mapper, err := apiutil.NewDynamicRESTMapper(cfg, httpClient) Expect(err).ToNot(HaveOccurred()) c, err := cache.New(cfg, cache.Options{Mapper: mapper}) diff --git a/pkg/cache/informer_cache_unit_test.go b/pkg/cache/informer_cache_unit_test.go index 6f66e4bd89..4772223748 100644 --- a/pkg/cache/informer_cache_unit_test.go +++ b/pkg/cache/informer_cache_unit_test.go @@ -17,10 +17,11 @@ limitations under the License. package cache import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,7 +39,8 @@ const ( var _ = Describe("ip.objectTypeForListObject", func() { ip := &informerCache{ - InformersMap: &internal.InformersMap{Scheme: scheme.Scheme}, + scheme: scheme.Scheme, + Informers: &internal.Informers{}, } It("should find the object type for unstructured lists", func() { @@ -54,7 +56,21 @@ var _ = Describe("ip.objectTypeForListObject", func() { referenceUnstructured := &unstructured.Unstructured{} referenceUnstructured.SetGroupVersionKind(*gvk) Expect(obj).To(Equal(referenceUnstructured)) + }) + + It("should find the object type for partial object metadata lists", func() { + partialList := &metav1.PartialObjectMetadataList{} + partialList.APIVersion = ("v1") + partialList.Kind = "PodList" + gvk, obj, err := ip.objectTypeForListObject(partialList) + Expect(err).ToNot(HaveOccurred()) + Expect(gvk.Group).To(Equal("")) + Expect(gvk.Version).To(Equal("v1")) + Expect(gvk.Kind).To(Equal("Pod")) + referencePartial := &metav1.PartialObjectMetadata{} + referencePartial.SetGroupVersionKind(*gvk) + Expect(obj).To(Equal(referencePartial)) }) It("should find the object type of a list with a slice of literals items field", func() { @@ -63,22 +79,21 @@ var _ = Describe("ip.objectTypeForListObject", func() { Expect(gvk.Group).To(Equal("")) Expect(gvk.Version).To(Equal("v1")) Expect(gvk.Kind).To(Equal("Pod")) - var referencePod *corev1.Pod + referencePod := &corev1.Pod{} Expect(obj).To(Equal(referencePod)) - }) It("should find the object type of a list with a slice of pointers items field", func() { By("registering the type", func() { - ip.Scheme = runtime.NewScheme() + ip.scheme = runtime.NewScheme() err := (&crscheme.Builder{ GroupVersion: schema.GroupVersion{Group: itemPointerSliceTypeGroupName, Version: itemPointerSliceTypeVersion}, }). Register( &controllertest.UnconventionalListType{}, &controllertest.UnconventionalListTypeList{}, - ).AddToScheme(ip.Scheme) - Expect(err).To(BeNil()) + ).AddToScheme(ip.scheme) + Expect(err).ToNot(HaveOccurred()) }) By("calling objectTypeForListObject", func() { @@ -87,7 +102,7 @@ var _ = Describe("ip.objectTypeForListObject", func() { Expect(gvk.Group).To(Equal(itemPointerSliceTypeGroupName)) Expect(gvk.Version).To(Equal(itemPointerSliceTypeVersion)) Expect(gvk.Kind).To(Equal("UnconventionalListType")) - var referenceObject *controllertest.UnconventionalListType + referenceObject := &controllertest.UnconventionalListType{} Expect(obj).To(Equal(referenceObject)) }) }) diff --git a/pkg/cache/informertest/fake_cache.go b/pkg/cache/informertest/fake_cache.go index e115d380f7..a1a442316f 100644 --- a/pkg/cache/informertest/fake_cache.go +++ b/pkg/cache/informertest/fake_cache.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" @@ -39,7 +40,7 @@ type FakeInformers struct { } // GetInformerForKind implements Informers. -func (c *FakeInformers) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (cache.Informer, error) { +func (c *FakeInformers) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...cache.InformerGetOption) (cache.Informer, error) { if c.Scheme == nil { c.Scheme = scheme.Scheme } @@ -52,14 +53,7 @@ func (c *FakeInformers) GetInformerForKind(ctx context.Context, gvk schema.Group // FakeInformerForKind implements Informers. func (c *FakeInformers) FakeInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (*controllertest.FakeInformer, error) { - if c.Scheme == nil { - c.Scheme = scheme.Scheme - } - obj, err := c.Scheme.New(gvk) - if err != nil { - return nil, err - } - i, err := c.informerFor(gvk, obj) + i, err := c.GetInformerForKind(ctx, gvk) if err != nil { return nil, err } @@ -67,7 +61,7 @@ func (c *FakeInformers) FakeInformerForKind(ctx context.Context, gvk schema.Grou } // GetInformer implements Informers. -func (c *FakeInformers) GetInformer(ctx context.Context, obj client.Object) (cache.Informer, error) { +func (c *FakeInformers) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { if c.Scheme == nil { c.Scheme = scheme.Scheme } @@ -79,6 +73,20 @@ func (c *FakeInformers) GetInformer(ctx context.Context, obj client.Object) (cac return c.informerFor(gvk, obj) } +// RemoveInformer implements Informers. +func (c *FakeInformers) RemoveInformer(ctx context.Context, obj client.Object) error { + if c.Scheme == nil { + c.Scheme = scheme.Scheme + } + gvks, _, err := c.Scheme.ObjectKinds(obj) + if err != nil { + return err + } + gvk := gvks[0] + delete(c.InformersByGVK, gvk) + return nil +} + // WaitForCacheSync implements Informers. func (c *FakeInformers) WaitForCacheSync(ctx context.Context) bool { if c.Synced == nil { @@ -88,16 +96,8 @@ func (c *FakeInformers) WaitForCacheSync(ctx context.Context) bool { } // FakeInformerFor implements Informers. -func (c *FakeInformers) FakeInformerFor(obj runtime.Object) (*controllertest.FakeInformer, error) { - if c.Scheme == nil { - c.Scheme = scheme.Scheme - } - gvks, _, err := c.Scheme.ObjectKinds(obj) - if err != nil { - return nil, err - } - gvk := gvks[0] - i, err := c.informerFor(gvk, obj) +func (c *FakeInformers) FakeInformerFor(ctx context.Context, obj client.Object) (*controllertest.FakeInformer, error) { + i, err := c.GetInformer(ctx, obj) if err != nil { return nil, err } @@ -131,7 +131,7 @@ func (c *FakeInformers) IndexField(ctx context.Context, obj client.Object, field } // Get implements Cache. -func (c *FakeInformers) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { +func (c *FakeInformers) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { return nil } diff --git a/pkg/cache/internal/cache_reader.go b/pkg/cache/internal/cache_reader.go index b95af18d78..eb6b544855 100644 --- a/pkg/cache/internal/cache_reader.go +++ b/pkg/cache/internal/cache_reader.go @@ -27,16 +27,16 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" ) // CacheReader is a client.Reader. var _ client.Reader = &CacheReader{} -// CacheReader wraps a cache.Index to implement the client.CacheReader interface for a single type. +// CacheReader wraps a cache.Index to implement the client.Reader interface for a single type. type CacheReader struct { // indexer is the underlying indexer wrapped by this cache. indexer cache.Indexer @@ -54,7 +54,10 @@ type CacheReader struct { } // Get checks the indexer for the object and writes a copy of it if found. -func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object) error { +func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error { + getOpts := client.GetOptions{} + getOpts.ApplyOptions(opts) + if c.scopeName == apimeta.RESTScopeNameRoot { key.Namespace = "" } @@ -68,9 +71,9 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob // Not found, return an error if !exists { - // Resource gets transformed into Kind in the error anyway, so this is fine return apierrors.NewNotFound(schema.GroupResource{ - Group: c.groupVersionKind.Group, + Group: c.groupVersionKind.Group, + // Resource gets set as Kind in the error so this is fine Resource: c.groupVersionKind.Kind, }, key.Name) } @@ -81,7 +84,7 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob return fmt.Errorf("cache contained %T, which is not an Object", obj) } - if c.disableDeepCopy { + if c.disableDeepCopy || (getOpts.UnsafeDisableDeepCopy != nil && *getOpts.UnsafeDisableDeepCopy) { // skip deep copy which might be unsafe // you must DeepCopy any object before mutating it outside } else { @@ -97,7 +100,7 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob return fmt.Errorf("cache had type %s, but %s was asked for", objVal.Type(), outVal.Type()) } reflect.Indirect(outVal).Set(reflect.Indirect(objVal)) - if !c.disableDeepCopy { + if !c.disableDeepCopy && (getOpts.UnsafeDisableDeepCopy == nil || !*getOpts.UnsafeDisableDeepCopy) { out.GetObjectKind().SetGroupVersionKind(c.groupVersionKind) } @@ -112,18 +115,20 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) + if listOpts.Continue != "" { + return fmt.Errorf("continue list option is not supported by the cache") + } + switch { case listOpts.FieldSelector != nil: - // TODO(directxman12): support more complicated field selectors by - // combining multiple indices, GetIndexers, etc - field, val, requiresExact := requiresExactMatch(listOpts.FieldSelector) + requiresExact := selector.RequiresExactMatch(listOpts.FieldSelector) if !requiresExact { return fmt.Errorf("non-exact field matches are not supported by the cache") } - // list all objects by the field selector. If this is namespaced and we have one, ask for the - // namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces" + // list all objects by the field selector. If this is namespaced and we have one, ask for the + // namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces" // namespace. - objs, err = c.indexer.ByIndex(FieldIndexName(field), KeyToNamespacedKey(listOpts.Namespace, val)) + objs, err = byIndexes(c.indexer, listOpts.FieldSelector.Requirements(), listOpts.Namespace) case listOpts.Namespace != "": objs, err = c.indexer.ByIndex(cache.NamespaceIndex, listOpts.Namespace) default: @@ -148,7 +153,7 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli } obj, isObj := item.(runtime.Object) if !isObj { - return fmt.Errorf("cache contained %T, which is not an Object", obj) + return fmt.Errorf("cache contained %T, which is not an Object", item) } meta, err := apimeta.Accessor(obj) if err != nil { @@ -162,7 +167,7 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli } var outObj runtime.Object - if c.disableDeepCopy { + if c.disableDeepCopy || (listOpts.UnsafeDisableDeepCopy != nil && *listOpts.UnsafeDisableDeepCopy) { // skip deep copy which might be unsafe // you must DeepCopy any object before mutating it outside outObj = obj @@ -172,11 +177,65 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli } runtimeObjs = append(runtimeObjs, outObj) } - return apimeta.SetList(out, runtimeObjs) + + if err := apimeta.SetList(out, runtimeObjs); err != nil { + return err + } + + out.SetContinue("continue-not-supported") + return nil +} + +func byIndexes(indexer cache.Indexer, requires fields.Requirements, namespace string) ([]interface{}, error) { + var ( + err error + objs []interface{} + vals []string + ) + indexers := indexer.GetIndexers() + for idx, req := range requires { + indexName := FieldIndexName(req.Field) + indexedValue := KeyToNamespacedKey(namespace, req.Value) + if idx == 0 { + // we use first require to get snapshot data + // TODO(halfcrazy): use complicated index when client-go provides byIndexes + // https://github.com/kubernetes/kubernetes/issues/109329 + objs, err = indexer.ByIndex(indexName, indexedValue) + if err != nil { + return nil, err + } + if len(objs) == 0 { + return nil, nil + } + continue + } + fn, exist := indexers[indexName] + if !exist { + return nil, fmt.Errorf("index with name %s does not exist", indexName) + } + filteredObjects := make([]interface{}, 0, len(objs)) + for _, obj := range objs { + vals, err = fn(obj) + if err != nil { + return nil, err + } + for _, val := range vals { + if val == indexedValue { + filteredObjects = append(filteredObjects, obj) + break + } + } + } + if len(filteredObjects) == 0 { + return nil, nil + } + objs = filteredObjects + } + return objs, nil } // objectKeyToStorageKey converts an object key to store key. -// It's akin to MetaNamespaceKeyFunc. It's separate from +// It's akin to MetaNamespaceKeyFunc. It's separate from // String to allow keeping the key format easily in sync with // MetaNamespaceKeyFunc. func objectKeyToStoreKey(k client.ObjectKey) string { @@ -186,26 +245,13 @@ func objectKeyToStoreKey(k client.ObjectKey) string { return k.Namespace + "/" + k.Name } -// requiresExactMatch checks if the given field selector is of the form `k=v` or `k==v`. -func requiresExactMatch(sel fields.Selector) (field, val string, required bool) { - reqs := sel.Requirements() - if len(reqs) != 1 { - return "", "", false - } - req := reqs[0] - if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { - return "", "", false - } - return req.Field, req.Value, true -} - // FieldIndexName constructs the name of the index over the given field, // for use with an indexer. func FieldIndexName(field string) string { return "field:" + field } -// noNamespaceNamespace is used as the "namespace" when we want to list across all namespaces. +// allNamespacesNamespace is used as the "namespace" when we want to list across all namespaces. const allNamespacesNamespace = "__all_namespaces" // KeyToNamespacedKey prefixes the given index key with a namespace diff --git a/pkg/cache/internal/deleg_map.go b/pkg/cache/internal/deleg_map.go deleted file mode 100644 index 9bfc8463fd..0000000000 --- a/pkg/cache/internal/deleg_map.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import ( - "context" - "time" - - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" -) - -// InformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. -// It uses a standard parameter codec constructed based on the given generated Scheme. -type InformersMap struct { - // we abstract over the details of structured/unstructured/metadata with the specificInformerMaps - // TODO(directxman12): genericize this over different projections now that we have 3 different maps - - structured *specificInformersMap - unstructured *specificInformersMap - metadata *specificInformersMap - - // Scheme maps runtime.Objects to GroupVersionKinds - Scheme *runtime.Scheme -} - -// NewInformersMap creates a new InformersMap that can create informers for -// both structured and unstructured objects. -func NewInformersMap(config *rest.Config, - scheme *runtime.Scheme, - mapper meta.RESTMapper, - resync time.Duration, - namespace string, - selectors SelectorsByGVK, - disableDeepCopy DisableDeepCopyByGVK, -) *InformersMap { - return &InformersMap{ - structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy), - unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy), - metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy), - - Scheme: scheme, - } -} - -// Start calls Run on each of the informers and sets started to true. Blocks on the context. -func (m *InformersMap) Start(ctx context.Context) error { - go m.structured.Start(ctx) - go m.unstructured.Start(ctx) - go m.metadata.Start(ctx) - <-ctx.Done() - return nil -} - -// WaitForCacheSync waits until all the caches have been started and synced. -func (m *InformersMap) WaitForCacheSync(ctx context.Context) bool { - syncedFuncs := append([]cache.InformerSynced(nil), m.structured.HasSyncedFuncs()...) - syncedFuncs = append(syncedFuncs, m.unstructured.HasSyncedFuncs()...) - syncedFuncs = append(syncedFuncs, m.metadata.HasSyncedFuncs()...) - - if !m.structured.waitForStarted(ctx) { - return false - } - if !m.unstructured.waitForStarted(ctx) { - return false - } - if !m.metadata.waitForStarted(ctx) { - return false - } - return cache.WaitForCacheSync(ctx.Done(), syncedFuncs...) -} - -// Get will create a new Informer and add it to the map of InformersMap if none exists. Returns -// the Informer from the map. -func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) { - switch obj.(type) { - case *unstructured.Unstructured: - return m.unstructured.Get(ctx, gvk, obj) - case *unstructured.UnstructuredList: - return m.unstructured.Get(ctx, gvk, obj) - case *metav1.PartialObjectMetadata: - return m.metadata.Get(ctx, gvk, obj) - case *metav1.PartialObjectMetadataList: - return m.metadata.Get(ctx, gvk, obj) - default: - return m.structured.Get(ctx, gvk, obj) - } -} - -// newStructuredInformersMap creates a new InformersMap for structured objects. -func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, - namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, createStructuredListWatch) -} - -// newUnstructuredInformersMap creates a new InformersMap for unstructured objects. -func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, - namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, createUnstructuredListWatch) -} - -// newMetadataInformersMap creates a new InformersMap for metadata-only objects. -func newMetadataInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, - namespace string, selectors SelectorsByGVK, disableDeepCopy DisableDeepCopyByGVK) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, disableDeepCopy, createMetadataListWatch) -} diff --git a/pkg/cache/internal/disabledeepcopy.go b/pkg/cache/internal/disabledeepcopy.go deleted file mode 100644 index 54bd7eec93..0000000000 --- a/pkg/cache/internal/disabledeepcopy.go +++ /dev/null @@ -1,35 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import "k8s.io/apimachinery/pkg/runtime/schema" - -// GroupVersionKindAll is the argument to represent all GroupVersionKind types. -var GroupVersionKindAll = schema.GroupVersionKind{} - -// DisableDeepCopyByGVK associate a GroupVersionKind to disable DeepCopy during get or list from cache. -type DisableDeepCopyByGVK map[schema.GroupVersionKind]bool - -// IsDisabled returns whether a GroupVersionKind is disabled DeepCopy. -func (disableByGVK DisableDeepCopyByGVK) IsDisabled(gvk schema.GroupVersionKind) bool { - if d, ok := disableByGVK[gvk]; ok { - return d - } else if d, ok = disableByGVK[GroupVersionKindAll]; ok { - return d - } - return false -} diff --git a/pkg/cache/internal/informers.go b/pkg/cache/internal/informers.go new file mode 100644 index 0000000000..f216be0d9e --- /dev/null +++ b/pkg/cache/internal/informers.go @@ -0,0 +1,616 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "errors" + "fmt" + "math/rand" + "net/http" + "sync" + "time" + + "github.com/go-logr/logr" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/metadata" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" + "sigs.k8s.io/controller-runtime/pkg/internal/syncs" +) + +var log = logf.RuntimeLog.WithName("cache") + +// InformersOpts configures an InformerMap. +type InformersOpts struct { + HTTPClient *http.Client + Scheme *runtime.Scheme + Mapper meta.RESTMapper + ResyncPeriod time.Duration + Namespace string + NewInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer + Selector Selector + Transform cache.TransformFunc + UnsafeDisableDeepCopy bool + EnableWatchBookmarks bool + WatchErrorHandler cache.WatchErrorHandlerWithContext +} + +// NewInformers creates a new InformersMap that can create informers under the hood. +func NewInformers(config *rest.Config, options *InformersOpts) *Informers { + newInformer := cache.NewSharedIndexInformer + if options.NewInformer != nil { + newInformer = options.NewInformer + } + return &Informers{ + config: config, + httpClient: options.HTTPClient, + scheme: options.Scheme, + mapper: options.Mapper, + tracker: tracker{ + Structured: make(map[schema.GroupVersionKind]*Cache), + Unstructured: make(map[schema.GroupVersionKind]*Cache), + Metadata: make(map[schema.GroupVersionKind]*Cache), + }, + codecs: serializer.NewCodecFactory(options.Scheme), + paramCodec: runtime.NewParameterCodec(options.Scheme), + resync: options.ResyncPeriod, + startWait: make(chan struct{}), + namespace: options.Namespace, + selector: options.Selector, + transform: options.Transform, + unsafeDisableDeepCopy: options.UnsafeDisableDeepCopy, + enableWatchBookmarks: options.EnableWatchBookmarks, + newInformer: newInformer, + watchErrorHandler: options.WatchErrorHandler, + } +} + +// Cache contains the cached data for an Cache. +type Cache struct { + // Informer is the cached informer + Informer cache.SharedIndexInformer + + // CacheReader wraps Informer and implements the CacheReader interface for a single type + Reader CacheReader + + // Stop can be used to stop this individual informer. + stop chan struct{} +} + +// Start starts the informer managed by a MapEntry. +// Blocks until the informer stops. The informer can be stopped +// either individually (via the entry's stop channel) or globally +// via the provided stop argument. +func (c *Cache) Start(stop <-chan struct{}) { + // Stop on either the whole map stopping or just this informer being removed. + internalStop, cancel := syncs.MergeChans(stop, c.stop) + defer cancel() + // Convert the stop channel to a context and then add the logger. + c.Informer.RunWithContext(logr.NewContext(wait.ContextForChannel(internalStop), log)) +} + +type tracker struct { + Structured map[schema.GroupVersionKind]*Cache + Unstructured map[schema.GroupVersionKind]*Cache + Metadata map[schema.GroupVersionKind]*Cache +} + +// GetOptions provides configuration to customize the behavior when +// getting an informer. +type GetOptions struct { + // BlockUntilSynced controls if the informer retrieval will block until the informer is synced. Defaults to `true`. + BlockUntilSynced *bool +} + +// Informers create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. +// It uses a standard parameter codec constructed based on the given generated Scheme. +type Informers struct { + // httpClient is used to create a new REST client + httpClient *http.Client + + // scheme maps runtime.Objects to GroupVersionKinds + scheme *runtime.Scheme + + // config is used to talk to the apiserver + config *rest.Config + + // mapper maps GroupVersionKinds to Resources + mapper meta.RESTMapper + + // tracker tracks informers keyed by their type and groupVersionKind + tracker tracker + + // codecs is used to create a new REST client + codecs serializer.CodecFactory + + // paramCodec is used by list and watch + paramCodec runtime.ParameterCodec + + // resync is the base frequency the informers are resynced + // a 10 percent jitter will be added to the resync period between informers + // so that all informers will not send list requests simultaneously. + resync time.Duration + + // mu guards access to the map + mu sync.RWMutex + + // started is true if the informers have been started + started bool + + // startWait is a channel that is closed after the + // informer has been started. + startWait chan struct{} + + // waitGroup is the wait group that is used to wait for all informers to stop + waitGroup sync.WaitGroup + + // stopped is true if the informers have been stopped + stopped bool + + // ctx is the context to stop informers + ctx context.Context + + // namespace is the namespace that all ListWatches are restricted to + // default or empty string means all namespaces + namespace string + + selector Selector + transform cache.TransformFunc + unsafeDisableDeepCopy bool + enableWatchBookmarks bool + + // NewInformer allows overriding of the shared index informer constructor for testing. + newInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer + + // watchErrorHandler allows the shared index informer's + // watchErrorHandler to be set by overriding the options + // or to use the default watchErrorHandler + watchErrorHandler cache.WatchErrorHandlerWithContext +} + +// Start calls Run on each of the informers and sets started to true. Blocks on the context. +// It doesn't return start because it can't return an error, and it's not a runnable directly. +func (ip *Informers) Start(ctx context.Context) error { + if err := func() error { + ip.mu.Lock() + defer ip.mu.Unlock() + + if ip.started { + return errors.New("informer already started") //nolint:stylecheck + } + + // Set the context so it can be passed to informers that are added later + ip.ctx = ctx + + // Start each informer + for _, i := range ip.tracker.Structured { + ip.startInformerLocked(i) + } + for _, i := range ip.tracker.Unstructured { + ip.startInformerLocked(i) + } + for _, i := range ip.tracker.Metadata { + ip.startInformerLocked(i) + } + + // Set started to true so we immediately start any informers added later. + ip.started = true + close(ip.startWait) + + return nil + }(); err != nil { + return err + } + <-ctx.Done() // Block until the context is done + ip.mu.Lock() + ip.stopped = true // Set stopped to true so we don't start any new informers + ip.mu.Unlock() + ip.waitGroup.Wait() // Block until all informers have stopped + return nil +} + +func (ip *Informers) startInformerLocked(cacheEntry *Cache) { + // Don't start the informer in case we are already waiting for the items in + // the waitGroup to finish, since waitGroups don't support waiting and adding + // at the same time. + if ip.stopped { + return + } + + ip.waitGroup.Add(1) + go func() { + defer ip.waitGroup.Done() + cacheEntry.Start(ip.ctx.Done()) + }() +} + +func (ip *Informers) waitForStarted(ctx context.Context) bool { + select { + case <-ip.startWait: + return true + case <-ctx.Done(): + return false + } +} + +// getHasSyncedFuncs returns all the HasSynced functions for the informers in this map. +func (ip *Informers) getHasSyncedFuncs() []cache.InformerSynced { + ip.mu.RLock() + defer ip.mu.RUnlock() + + res := make([]cache.InformerSynced, 0, + len(ip.tracker.Structured)+len(ip.tracker.Unstructured)+len(ip.tracker.Metadata), + ) + for _, i := range ip.tracker.Structured { + res = append(res, i.Informer.HasSynced) + } + for _, i := range ip.tracker.Unstructured { + res = append(res, i.Informer.HasSynced) + } + for _, i := range ip.tracker.Metadata { + res = append(res, i.Informer.HasSynced) + } + return res +} + +// WaitForCacheSync waits until all the caches have been started and synced. +func (ip *Informers) WaitForCacheSync(ctx context.Context) bool { + if !ip.waitForStarted(ctx) { + return false + } + return cache.WaitForCacheSync(ctx.Done(), ip.getHasSyncedFuncs()...) +} + +// Peek attempts to get the informer for the GVK, but does not start one if one does not exist. +func (ip *Informers) Peek(gvk schema.GroupVersionKind, obj runtime.Object) (res *Cache, started bool, ok bool) { + ip.mu.RLock() + defer ip.mu.RUnlock() + i, ok := ip.informersByType(obj)[gvk] + return i, ip.started, ok +} + +// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns +// the Informer from the map. +func (ip *Informers) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object, opts *GetOptions) (bool, *Cache, error) { + // Return the informer if it is found + i, started, ok := ip.Peek(gvk, obj) + if !ok { + var err error + if i, started, err = ip.addInformerToMap(gvk, obj); err != nil { + return started, nil, err + } + } + + shouldBlock := true + if opts.BlockUntilSynced != nil { + shouldBlock = *opts.BlockUntilSynced + } + + if shouldBlock && started && !i.Informer.HasSynced() { + // Wait for it to sync before returning the Informer so that folks don't read from a stale cache. + if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) { + return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0) + } + } + + return started, i, nil +} + +// Remove removes an informer entry and stops it if it was running. +func (ip *Informers) Remove(gvk schema.GroupVersionKind, obj runtime.Object) { + ip.mu.Lock() + defer ip.mu.Unlock() + + informerMap := ip.informersByType(obj) + + entry, ok := informerMap[gvk] + if !ok { + return + } + close(entry.stop) + delete(informerMap, gvk) +} + +func (ip *Informers) informersByType(obj runtime.Object) map[schema.GroupVersionKind]*Cache { + switch obj.(type) { + case runtime.Unstructured: + return ip.tracker.Unstructured + case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList: + return ip.tracker.Metadata + default: + return ip.tracker.Structured + } +} + +// addInformerToMap either returns an existing informer or creates a new informer, adds it to the map and returns it. +func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.Object) (*Cache, bool, error) { + ip.mu.Lock() + defer ip.mu.Unlock() + + // Check the cache to see if we already have an Informer. If we do, return the Informer. + // This is for the case where 2 routines tried to get the informer when it wasn't in the map + // so neither returned early, but the first one created it. + if i, ok := ip.informersByType(obj)[gvk]; ok { + return i, ip.started, nil + } + + // Create a NewSharedIndexInformer and add it to the map. + listWatcher, err := ip.makeListWatcher(gvk, obj) + if err != nil { + return nil, false, err + } + sharedIndexInformer := ip.newInformer(&cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + ip.selector.ApplyToList(&opts) + return listWatcher.ListWithContextFunc(ctx, opts) + }, + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + + ip.selector.ApplyToList(&opts) + return listWatcher.WatchFuncWithContext(ctx, opts) + }, + }, obj, calculateResyncPeriod(ip.resync), cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, + }) + + // Set WatchErrorHandler on SharedIndexInformer if set + if ip.watchErrorHandler != nil { + if err := sharedIndexInformer.SetWatchErrorHandlerWithContext(ip.watchErrorHandler); err != nil { + return nil, false, err + } + } + + // Check to see if there is a transformer for this gvk + if err := sharedIndexInformer.SetTransform(ip.transform); err != nil { + return nil, false, err + } + + mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, false, err + } + + // Create the new entry and set it in the map. + i := &Cache{ + Informer: sharedIndexInformer, + Reader: CacheReader{ + indexer: sharedIndexInformer.GetIndexer(), + groupVersionKind: gvk, + scopeName: mapping.Scope.Name(), + disableDeepCopy: ip.unsafeDisableDeepCopy, + }, + stop: make(chan struct{}), + } + ip.informersByType(obj)[gvk] = i + + // Start the informer in case the InformersMap has started, otherwise it will be + // started when the InformersMap starts. + if ip.started { + ip.startInformerLocked(i) + } + return i, ip.started, nil +} + +func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Object) (*cache.ListWatch, error) { + // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the + // groupVersionKind to the Resource API we will use. + mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, err + } + + // Figure out if the GVK we're dealing with is global, or namespace scoped. + var namespace string + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + namespace = restrictNamespaceBySelector(ip.namespace, ip.selector) + } + + switch obj.(type) { + // + // Unstructured + // + case runtime.Unstructured: + // If the rest configuration has a negotiated serializer passed in, + // we should remove it and use the one that the dynamic client sets for us. + cfg := rest.CopyConfig(ip.config) + cfg.NegotiatedSerializer = nil + dynamicClient, err := dynamic.NewForConfigAndClient(cfg, ip.httpClient) + if err != nil { + return nil, err + } + resources := dynamicClient.Resource(mapping.Resource) + return &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + if namespace != "" { + return resources.Namespace(namespace).List(ctx, opts) + } + return resources.List(ctx, opts) + }, + // Setup the watch function + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + + if namespace != "" { + return resources.Namespace(namespace).Watch(ctx, opts) + } + return resources.Watch(ctx, opts) + }, + }, nil + // + // Metadata + // + case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList: + // Always clear the negotiated serializer and use the one + // set from the metadata client. + cfg := rest.CopyConfig(ip.config) + cfg.NegotiatedSerializer = nil + + // Grab the metadata metadataClient. + metadataClient, err := metadata.NewForConfigAndClient(cfg, ip.httpClient) + if err != nil { + return nil, err + } + resources := metadataClient.Resource(mapping.Resource) + + return &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + var ( + list *metav1.PartialObjectMetadataList + err error + ) + if namespace != "" { + list, err = resources.Namespace(namespace).List(ctx, opts) + } else { + list, err = resources.List(ctx, opts) + } + if list != nil { + for i := range list.Items { + list.Items[i].SetGroupVersionKind(gvk) + } + } + return list, err + }, + // Setup the watch function + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watcher watch.Interface, err error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + + if namespace != "" { + watcher, err = resources.Namespace(namespace).Watch(ctx, opts) + } else { + watcher, err = resources.Watch(ctx, opts) + } + if err != nil { + return nil, err + } + return newGVKFixupWatcher(gvk, watcher), nil + }, + }, nil + // + // Structured. + // + default: + client, err := apiutil.RESTClientForGVK(gvk, false, false, ip.config, ip.codecs, ip.httpClient) + if err != nil { + return nil, err + } + listGVK := gvk.GroupVersion().WithKind(gvk.Kind + "List") + listObj, err := ip.scheme.New(listGVK) + if err != nil { + return nil, err + } + return &cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + // Build the request. + req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec) + if namespace != "" { + req.Namespace(namespace) + } + + // Create the resulting object, and execute the request. + res := listObj.DeepCopyObject() + if err := req.Do(ctx).Into(res); err != nil { + return nil, err + } + return res, nil + }, + // Setup the watch function + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + + // Build the request. + req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec) + if namespace != "" { + req.Namespace(namespace) + } + // Call the watch. + return req.Watch(ctx) + }, + }, nil + } +} + +// newGVKFixupWatcher adds a wrapper that preserves the GVK information when +// events come in. +// +// This works around a bug where GVK information is not passed into mapping +// functions when using the OnlyMetadata option in the builder. +// This issue is most likely caused by kubernetes/kubernetes#80609. +// See kubernetes-sigs/controller-runtime#1484. +// +// This was originally implemented as a cache.ResourceEventHandler wrapper but +// that contained a data race which was resolved by setting the GVK in a watch +// wrapper, before the objects are written to the cache. +// See kubernetes-sigs/controller-runtime#1650. +// +// The original watch wrapper was found to be incompatible with +// k8s.io/client-go/tools/cache.Reflector so it has been re-implemented as a +// watch.Filter which is compatible. +// See kubernetes-sigs/controller-runtime#1789. +func newGVKFixupWatcher(gvk schema.GroupVersionKind, watcher watch.Interface) watch.Interface { + return watch.Filter( + watcher, + func(in watch.Event) (watch.Event, bool) { + in.Object.GetObjectKind().SetGroupVersionKind(gvk) + return in, true + }, + ) +} + +// calculateResyncPeriod returns a duration based on the desired input +// this is so that multiple controllers don't get into lock-step and all +// hammer the apiserver with list requests simultaneously. +func calculateResyncPeriod(resync time.Duration) time.Duration { + // the factor will fall into [0.9, 1.1) + factor := rand.Float64()/5.0 + 0.9 + return time.Duration(float64(resync.Nanoseconds()) * factor) +} + +// restrictNamespaceBySelector returns either a global restriction for all ListWatches +// if not default/empty, or the namespace that a ListWatch for the specific resource +// is restricted to, based on a specified field selector for metadata.namespace field. +func restrictNamespaceBySelector(namespaceOpt string, s Selector) string { + if namespaceOpt != "" { + // namespace is already restricted + return namespaceOpt + } + fieldSelector := s.Field + if fieldSelector == nil || fieldSelector.Empty() { + return "" + } + // check whether a selector includes the namespace field + value, found := fieldSelector.RequiresExactMatch("metadata.namespace") + if found { + return value + } + return "" +} diff --git a/pkg/cache/internal/informers_map.go b/pkg/cache/internal/informers_map.go deleted file mode 100644 index f8e957343f..0000000000 --- a/pkg/cache/internal/informers_map.go +++ /dev/null @@ -1,478 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import ( - "context" - "fmt" - "math/rand" - "sync" - "time" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/metadata" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" -) - -func init() { - rand.Seed(time.Now().UnixNano()) -} - -// clientListWatcherFunc knows how to create a ListWatcher. -type createListWatcherFunc func(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) - -// newSpecificInformersMap returns a new specificInformersMap (like -// the generical InformersMap, except that it doesn't implement WaitForCacheSync). -func newSpecificInformersMap(config *rest.Config, - scheme *runtime.Scheme, - mapper meta.RESTMapper, - resync time.Duration, - namespace string, - selectors SelectorsByGVK, - disableDeepCopy DisableDeepCopyByGVK, - createListWatcher createListWatcherFunc) *specificInformersMap { - ip := &specificInformersMap{ - config: config, - Scheme: scheme, - mapper: mapper, - informersByGVK: make(map[schema.GroupVersionKind]*MapEntry), - codecs: serializer.NewCodecFactory(scheme), - paramCodec: runtime.NewParameterCodec(scheme), - resync: resync, - startWait: make(chan struct{}), - createListWatcher: createListWatcher, - namespace: namespace, - selectors: selectors, - disableDeepCopy: disableDeepCopy, - } - return ip -} - -// MapEntry contains the cached data for an Informer. -type MapEntry struct { - // Informer is the cached informer - Informer cache.SharedIndexInformer - - // CacheReader wraps Informer and implements the CacheReader interface for a single type - Reader CacheReader -} - -// specificInformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. -// It uses a standard parameter codec constructed based on the given generated Scheme. -type specificInformersMap struct { - // Scheme maps runtime.Objects to GroupVersionKinds - Scheme *runtime.Scheme - - // config is used to talk to the apiserver - config *rest.Config - - // mapper maps GroupVersionKinds to Resources - mapper meta.RESTMapper - - // informersByGVK is the cache of informers keyed by groupVersionKind - informersByGVK map[schema.GroupVersionKind]*MapEntry - - // codecs is used to create a new REST client - codecs serializer.CodecFactory - - // paramCodec is used by list and watch - paramCodec runtime.ParameterCodec - - // stop is the stop channel to stop informers - stop <-chan struct{} - - // resync is the base frequency the informers are resynced - // a 10 percent jitter will be added to the resync period between informers - // so that all informers will not send list requests simultaneously. - resync time.Duration - - // mu guards access to the map - mu sync.RWMutex - - // start is true if the informers have been started - started bool - - // startWait is a channel that is closed after the - // informer has been started. - startWait chan struct{} - - // createClient knows how to create a client and a list object, - // and allows for abstracting over the particulars of structured vs - // unstructured objects. - createListWatcher createListWatcherFunc - - // namespace is the namespace that all ListWatches are restricted to - // default or empty string means all namespaces - namespace string - - // selectors are the label or field selectors that will be added to the - // ListWatch ListOptions. - selectors SelectorsByGVK - - // disableDeepCopy indicates not to deep copy objects during get or list objects. - disableDeepCopy DisableDeepCopyByGVK -} - -// Start calls Run on each of the informers and sets started to true. Blocks on the context. -// It doesn't return start because it can't return an error, and it's not a runnable directly. -func (ip *specificInformersMap) Start(ctx context.Context) { - func() { - ip.mu.Lock() - defer ip.mu.Unlock() - - // Set the stop channel so it can be passed to informers that are added later - ip.stop = ctx.Done() - - // Start each informer - for _, informer := range ip.informersByGVK { - go informer.Informer.Run(ctx.Done()) - } - - // Set started to true so we immediately start any informers added later. - ip.started = true - close(ip.startWait) - }() - <-ctx.Done() -} - -func (ip *specificInformersMap) waitForStarted(ctx context.Context) bool { - select { - case <-ip.startWait: - return true - case <-ctx.Done(): - return false - } -} - -// HasSyncedFuncs returns all the HasSynced functions for the informers in this map. -func (ip *specificInformersMap) HasSyncedFuncs() []cache.InformerSynced { - ip.mu.RLock() - defer ip.mu.RUnlock() - syncedFuncs := make([]cache.InformerSynced, 0, len(ip.informersByGVK)) - for _, informer := range ip.informersByGVK { - syncedFuncs = append(syncedFuncs, informer.Informer.HasSynced) - } - return syncedFuncs -} - -// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns -// the Informer from the map. -func (ip *specificInformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) { - // Return the informer if it is found - i, started, ok := func() (*MapEntry, bool, bool) { - ip.mu.RLock() - defer ip.mu.RUnlock() - i, ok := ip.informersByGVK[gvk] - return i, ip.started, ok - }() - - if !ok { - var err error - if i, started, err = ip.addInformerToMap(gvk, obj); err != nil { - return started, nil, err - } - } - - if started && !i.Informer.HasSynced() { - // Wait for it to sync before returning the Informer so that folks don't read from a stale cache. - if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) { - return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0) - } - } - - return started, i, nil -} - -func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.Object) (*MapEntry, bool, error) { - ip.mu.Lock() - defer ip.mu.Unlock() - - // Check the cache to see if we already have an Informer. If we do, return the Informer. - // This is for the case where 2 routines tried to get the informer when it wasn't in the map - // so neither returned early, but the first one created it. - if i, ok := ip.informersByGVK[gvk]; ok { - return i, ip.started, nil - } - - // Create a NewSharedIndexInformer and add it to the map. - var lw *cache.ListWatch - lw, err := ip.createListWatcher(gvk, ip) - if err != nil { - return nil, false, err - } - ni := cache.NewSharedIndexInformer(lw, obj, resyncPeriod(ip.resync)(), cache.Indexers{ - cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, - }) - rm, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, false, err - } - - i := &MapEntry{ - Informer: ni, - Reader: CacheReader{ - indexer: ni.GetIndexer(), - groupVersionKind: gvk, - scopeName: rm.Scope.Name(), - disableDeepCopy: ip.disableDeepCopy.IsDisabled(gvk), - }, - } - ip.informersByGVK[gvk] = i - - // Start the Informer if need by - // TODO(seans): write thorough tests and document what happens here - can you add indexers? - // can you add eventhandlers? - if ip.started { - go i.Informer.Run(ip.stop) - } - return i, ip.started, nil -} - -// newListWatch returns a new ListWatch object that can be used to create a SharedIndexInformer. -func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) { - // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the - // groupVersionKind to the Resource API we will use. - mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, err - } - - client, err := apiutil.RESTClientForGVK(gvk, false, ip.config, ip.codecs) - if err != nil { - return nil, err - } - listGVK := gvk.GroupVersion().WithKind(gvk.Kind + "List") - listObj, err := ip.Scheme.New(listGVK) - if err != nil { - return nil, err - } - - // TODO: the functions that make use of this ListWatch should be adapted to - // pass in their own contexts instead of relying on this fixed one here. - ctx := context.TODO() - // Create a new ListWatch for the obj - return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - ip.selectors[gvk].ApplyToList(&opts) - res := listObj.DeepCopyObject() - namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk]) - isNamespaceScoped := namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot - err := client.Get().NamespaceIfScoped(namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Do(ctx).Into(res) - return res, err - }, - // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.selectors[gvk].ApplyToList(&opts) - // Watch needs to be set to true separately - opts.Watch = true - namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk]) - isNamespaceScoped := namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot - return client.Get().NamespaceIfScoped(namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Watch(ctx) - }, - }, nil -} - -func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) { - // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the - // groupVersionKind to the Resource API we will use. - mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, err - } - - // If the rest configuration has a negotiated serializer passed in, - // we should remove it and use the one that the dynamic client sets for us. - cfg := rest.CopyConfig(ip.config) - cfg.NegotiatedSerializer = nil - dynamicClient, err := dynamic.NewForConfig(cfg) - if err != nil { - return nil, err - } - - // TODO: the functions that make use of this ListWatch should be adapted to - // pass in their own contexts instead of relying on this fixed one here. - ctx := context.TODO() - // Create a new ListWatch for the obj - return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - ip.selectors[gvk].ApplyToList(&opts) - namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk]) - if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - return dynamicClient.Resource(mapping.Resource).Namespace(namespace).List(ctx, opts) - } - return dynamicClient.Resource(mapping.Resource).List(ctx, opts) - }, - // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.selectors[gvk].ApplyToList(&opts) - // Watch needs to be set to true separately - opts.Watch = true - namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk]) - if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - return dynamicClient.Resource(mapping.Resource).Namespace(namespace).Watch(ctx, opts) - } - return dynamicClient.Resource(mapping.Resource).Watch(ctx, opts) - }, - }, nil -} - -func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) { - // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the - // groupVersionKind to the Resource API we will use. - mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, err - } - - // Always clear the negotiated serializer and use the one - // set from the metadata client. - cfg := rest.CopyConfig(ip.config) - cfg.NegotiatedSerializer = nil - - // grab the metadata client - client, err := metadata.NewForConfig(cfg) - if err != nil { - return nil, err - } - - // TODO: the functions that make use of this ListWatch should be adapted to - // pass in their own contexts instead of relying on this fixed one here. - ctx := context.TODO() - - // create the relevant listwatch - return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - ip.selectors[gvk].ApplyToList(&opts) - - var ( - list *metav1.PartialObjectMetadataList - err error - ) - namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk]) - if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - list, err = client.Resource(mapping.Resource).Namespace(namespace).List(ctx, opts) - } else { - list, err = client.Resource(mapping.Resource).List(ctx, opts) - } - if list != nil { - for i := range list.Items { - list.Items[i].SetGroupVersionKind(gvk) - } - } - return list, err - }, - // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.selectors[gvk].ApplyToList(&opts) - // Watch needs to be set to true separately - opts.Watch = true - - var ( - watcher watch.Interface - err error - ) - namespace := restrictNamespaceBySelector(ip.namespace, ip.selectors[gvk]) - if namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - watcher, err = client.Resource(mapping.Resource).Namespace(namespace).Watch(ctx, opts) - } else { - watcher, err = client.Resource(mapping.Resource).Watch(ctx, opts) - } - if watcher != nil { - watcher = newGVKFixupWatcher(gvk, watcher) - } - return watcher, err - }, - }, nil -} - -type gvkFixupWatcher struct { - watcher watch.Interface - ch chan watch.Event - gvk schema.GroupVersionKind - wg sync.WaitGroup -} - -func newGVKFixupWatcher(gvk schema.GroupVersionKind, watcher watch.Interface) watch.Interface { - ch := make(chan watch.Event) - w := &gvkFixupWatcher{ - gvk: gvk, - watcher: watcher, - ch: ch, - } - w.wg.Add(1) - go w.run() - return w -} - -func (w *gvkFixupWatcher) run() { - for e := range w.watcher.ResultChan() { - e.Object.GetObjectKind().SetGroupVersionKind(w.gvk) - w.ch <- e - } - w.wg.Done() -} - -func (w *gvkFixupWatcher) Stop() { - w.watcher.Stop() - w.wg.Wait() - close(w.ch) -} - -func (w *gvkFixupWatcher) ResultChan() <-chan watch.Event { - return w.ch -} - -// resyncPeriod returns a function which generates a duration each time it is -// invoked; this is so that multiple controllers don't get into lock-step and all -// hammer the apiserver with list requests simultaneously. -func resyncPeriod(resync time.Duration) func() time.Duration { - return func() time.Duration { - // the factor will fall into [0.9, 1.1) - factor := rand.Float64()/5.0 + 0.9 //nolint:gosec - return time.Duration(float64(resync.Nanoseconds()) * factor) - } -} - -// restrictNamespaceBySelector returns either a global restriction for all ListWatches -// if not default/empty, or the namespace that a ListWatch for the specific resource -// is restricted to, based on a specified field selector for metadata.namespace field. -func restrictNamespaceBySelector(namespaceOpt string, s Selector) string { - if namespaceOpt != "" { - // namespace is already restricted - return namespaceOpt - } - fieldSelector := s.Field - if fieldSelector == nil || fieldSelector.Empty() { - return "" - } - // check whether a selector includes the namespace field - value, found := fieldSelector.RequiresExactMatch("metadata.namespace") - if found { - return value - } - return "" -} diff --git a/pkg/cache/internal/informers_test.go b/pkg/cache/internal/informers_test.go new file mode 100644 index 0000000000..854a39c1f1 --- /dev/null +++ b/pkg/cache/internal/informers_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" +) + +// Test that gvkFixupWatcher behaves like watch.FakeWatcher +// and that it overrides the GVK. +// These tests are adapted from the watch.FakeWatcher tests in: +// https://github.com/kubernetes/kubernetes/blob/adbda068c1808fcc8a64a94269e0766b5c46ec41/staging/src/k8s.io/apimachinery/pkg/watch/watch_test.go#L33-L78 +var _ = Describe("gvkFixupWatcher", func() { + It("behaves like watch.FakeWatcher", func() { + newTestType := func(name string) runtime.Object { + return &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + } + + f := watch.NewFake() + // This is the GVK which we expect the wrapper to set on all the events + expectedGVK := schema.GroupVersionKind{ + Group: "testgroup", + Version: "v1test2", + Kind: "TestKind", + } + gvkfw := newGVKFixupWatcher(expectedGVK, f) + + table := []struct { + t watch.EventType + s runtime.Object + }{ + {watch.Added, newTestType("foo")}, + {watch.Modified, newTestType("qux")}, + {watch.Modified, newTestType("bar")}, + {watch.Deleted, newTestType("bar")}, + {watch.Error, newTestType("error: blah")}, + } + + consumer := func(w watch.Interface) { + for _, expect := range table { + By(fmt.Sprintf("Fixing up watch.EventType: %v and passing it on", expect.t)) + got, ok := <-w.ResultChan() + Expect(ok).To(BeTrue(), "closed early") + Expect(expect.t).To(Equal(got.Type), "unexpected Event.Type or out-of-order Event") + Expect(got.Object).To(BeAssignableToTypeOf(&metav1.PartialObjectMetadata{}), "unexpected Event.Object type") + a := got.Object.(*metav1.PartialObjectMetadata) + Expect(got.Object.GetObjectKind().GroupVersionKind()).To(Equal(expectedGVK), "GVK was not fixed up") + expected := expect.s.DeepCopyObject() + expected.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + actual := a.DeepCopyObject() + actual.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + Expect(actual).To(Equal(expected), "unexpected change to the Object") + } + Eventually(w.ResultChan()).Should(BeClosed()) + } + + sender := func() { + f.Add(newTestType("foo")) + f.Action(watch.Modified, newTestType("qux")) + f.Modify(newTestType("bar")) + f.Delete(newTestType("bar")) + f.Error(newTestType("error: blah")) + f.Stop() + } + + go sender() + consumer(gvkfw) + }) +}) diff --git a/pkg/runtime/inject/inject_suite_test.go b/pkg/cache/internal/internal_suite_test.go similarity index 68% rename from pkg/runtime/inject/inject_suite_test.go rename to pkg/cache/internal/internal_suite_test.go index 98cf79ab3b..25ec0f1dbc 100644 --- a/pkg/runtime/inject/inject_suite_test.go +++ b/pkg/cache/internal/internal_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,18 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package inject +package internal import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Runtime Injection Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Cache Internal Suite") } diff --git a/pkg/cache/internal/selector.go b/pkg/cache/internal/selector.go index cd9c580008..c674379b99 100644 --- a/pkg/cache/internal/selector.go +++ b/pkg/cache/internal/selector.go @@ -20,12 +20,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" ) -// SelectorsByGVK associate a GroupVersionKind to a field/label selector. -type SelectorsByGVK map[schema.GroupVersionKind]Selector - // Selector specify the label/field selector to fill in ListOptions. type Selector struct { Label labels.Selector diff --git a/pkg/cache/multi_namespace_cache.go b/pkg/cache/multi_namespace_cache.go index dc29651b01..d7d7b0e7c2 100644 --- a/pkg/cache/multi_namespace_cache.go +++ b/pkg/cache/multi_namespace_cache.go @@ -23,50 +23,42 @@ import ( corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/internal/objectutil" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) -// NewCacheFunc - Function for creating a new cache from the options and a rest config. -type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error) - // a new global namespaced cache to handle cluster scoped resources. const globalCache = "_cluster-scope" -// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache. -// This will scope the cache to a list of namespaces. Listing for all namespaces -// will list for all the namespaces that this knows about. By default this will create -// a global cache for cluster scoped resource. Note that this is not intended -// to be used for excluding namespaces, this is better done via a Predicate. Also note that -// you may face performance issues when using this with a high number of namespaces. -func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc { - return func(config *rest.Config, opts Options) (Cache, error) { - opts, err := defaultOpts(config, opts) - if err != nil { - return nil, err - } - - caches := map[string]Cache{} +func newMultiNamespaceCache( + newCache newCacheFunc, + scheme *runtime.Scheme, + restMapper apimeta.RESTMapper, + namespaces map[string]Config, + globalConfig *Config, // may be nil in which case no cache for cluster-scoped objects will be created +) Cache { + // Create every namespace cache. + caches := map[string]Cache{} + for namespace, config := range namespaces { + caches[namespace] = newCache(config, namespace) + } - // create a cache for cluster scoped resources - gCache, err := New(config, opts) - if err != nil { - return nil, fmt.Errorf("error creating global cache %v", err) - } + // Create a cache for cluster scoped resources if requested + var clusterCache Cache + if globalConfig != nil { + clusterCache = newCache(*globalConfig, corev1.NamespaceAll) + } - for _, ns := range namespaces { - opts.Namespace = ns - c, err := New(config, opts) - if err != nil { - return nil, err - } - caches[ns] = c - } - return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil + return &multiNamespaceCache{ + namespaceToCache: caches, + Scheme: scheme, + RESTMapper: restMapper, + clusterCache: clusterCache, } } @@ -75,117 +67,148 @@ func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc { // operator to a list of namespaces instead of watching every namespace // in the cluster. type multiNamespaceCache struct { - namespaceToCache map[string]Cache Scheme *runtime.Scheme RESTMapper apimeta.RESTMapper + namespaceToCache map[string]Cache clusterCache Cache } var _ Cache = &multiNamespaceCache{} // Methods for multiNamespaceCache to conform to the Informers interface. -func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) { - informers := map[string]Informer{} - // If the object is clusterscoped, get the informer from clusterCache, +func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) { + // If the object is cluster scoped, get the informer from clusterCache, // if not use the namespaced caches. - isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper) + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) if err != nil { return nil, err } if !isNamespaced { - clusterCacheInf, err := c.clusterCache.GetInformer(ctx, obj) + clusterCacheInformer, err := c.clusterCache.GetInformer(ctx, obj, opts...) if err != nil { return nil, err } - informers[globalCache] = clusterCacheInf - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{ + namespaceToInformer: map[string]Informer{ + globalCache: clusterCacheInformer, + }, + }, nil } + namespaceToInformer := map[string]Informer{} for ns, cache := range c.namespaceToCache { - informer, err := cache.GetInformer(ctx, obj) + informer, err := cache.GetInformer(ctx, obj, opts...) if err != nil { return nil, err } - informers[ns] = informer + namespaceToInformer[ns] = informer } - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil } -func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) { - informers := map[string]Informer{} - +func (c *multiNamespaceCache) RemoveInformer(ctx context.Context, obj client.Object) error { // If the object is clusterscoped, get the informer from clusterCache, // if not use the namespaced caches. - isNamespaced, err := objectutil.IsAPINamespacedWithGVK(gvk, c.Scheme, c.RESTMapper) + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) + if err != nil { + return err + } + if !isNamespaced { + return c.clusterCache.RemoveInformer(ctx, obj) + } + + for _, cache := range c.namespaceToCache { + err := cache.RemoveInformer(ctx, obj) + if err != nil { + return err + } + } + + return nil +} + +func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) { + // If the object is cluster scoped, get the informer from clusterCache, + // if not use the namespaced caches. + isNamespaced, err := apiutil.IsGVKNamespaced(gvk, c.RESTMapper) if err != nil { return nil, err } if !isNamespaced { - clusterCacheInf, err := c.clusterCache.GetInformerForKind(ctx, gvk) + clusterCacheInformer, err := c.clusterCache.GetInformerForKind(ctx, gvk, opts...) if err != nil { return nil, err } - informers[globalCache] = clusterCacheInf - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{ + namespaceToInformer: map[string]Informer{ + globalCache: clusterCacheInformer, + }, + }, nil } + namespaceToInformer := map[string]Informer{} for ns, cache := range c.namespaceToCache { - informer, err := cache.GetInformerForKind(ctx, gvk) + informer, err := cache.GetInformerForKind(ctx, gvk, opts...) if err != nil { return nil, err } - informers[ns] = informer + namespaceToInformer[ns] = informer } - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil } func (c *multiNamespaceCache) Start(ctx context.Context) error { + errs := make(chan error) // start global cache - go func() { - err := c.clusterCache.Start(ctx) - if err != nil { - log.Error(err, "cluster scoped cache failed to start") - } - }() + if c.clusterCache != nil { + go func() { + err := c.clusterCache.Start(ctx) + if err != nil { + errs <- fmt.Errorf("failed to start cluster-scoped cache: %w", err) + } + }() + } // start namespaced caches for ns, cache := range c.namespaceToCache { go func(ns string, cache Cache) { - err := cache.Start(ctx) - if err != nil { - log.Error(err, "multinamespace cache failed to start namespaced informer", "namespace", ns) + if err := cache.Start(ctx); err != nil { + errs <- fmt.Errorf("failed to start cache for namespace %s: %w", ns, err) } }(ns, cache) } - - <-ctx.Done() - return nil + select { + case <-ctx.Done(): + return nil + case err := <-errs: + return err + } } func (c *multiNamespaceCache) WaitForCacheSync(ctx context.Context) bool { synced := true for _, cache := range c.namespaceToCache { - if s := cache.WaitForCacheSync(ctx); !s { - synced = s + if !cache.WaitForCacheSync(ctx) { + synced = false } } // check if cluster scoped cache has synced - if !c.clusterCache.WaitForCacheSync(ctx) { + if c.clusterCache != nil && !c.clusterCache.WaitForCacheSync(ctx) { synced = false } return synced } func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { - isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper) + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) if err != nil { - return nil //nolint:nilerr + return err } if !isNamespaced { @@ -200,8 +223,8 @@ func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object, return nil } -func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { - isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper) +func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) if err != nil { return err } @@ -213,9 +236,12 @@ func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj cache, ok := c.namespaceToCache[key.Namespace] if !ok { + if global, hasGlobal := c.namespaceToCache[metav1.NamespaceAll]; hasGlobal { + return global.Get(ctx, key, obj, opts...) + } return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", key) } - return cache.Get(ctx, key, obj) + return cache.Get(ctx, key, obj, opts...) } // List multi namespace cache will get all the objects in the namespaces that the cache is watching if asked for all namespaces. @@ -223,7 +249,11 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) - isNamespaced, err := objectutil.IsAPINamespaced(list, c.Scheme, c.RESTMapper) + if listOpts.Continue != "" { + return fmt.Errorf("continue list option is not supported by the cache") + } + + isNamespaced, err := apiutil.IsObjectNamespaced(list, c.Scheme, c.RESTMapper) if err != nil { return err } @@ -236,7 +266,10 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, if listOpts.Namespace != corev1.NamespaceAll { cache, ok := c.namespaceToCache[listOpts.Namespace] if !ok { - return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", listOpts.Namespace) + if global, hasGlobal := c.namespaceToCache[AllNamespaces]; hasGlobal { + return global.List(ctx, list, opts...) + } + return fmt.Errorf("unable to list: %v because of unknown namespace for the cache", listOpts.Namespace) } return cache.List(ctx, list, opts...) } @@ -246,10 +279,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, return err } - allItems, err := apimeta.ExtractList(list) - if err != nil { - return err - } + allItems := []runtime.Object{} limitSet := listOpts.Limit > 0 @@ -269,12 +299,14 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, return fmt.Errorf("object: %T must be a list type", list) } allItems = append(allItems, items...) + // The last list call should have the most correct resource version. resourceVersion = accessor.GetResourceVersion() if limitSet { // decrement Limit by the number of items // fetched from the current namespace. listOpts.Limit -= int64(len(items)) + // if a Limit was set and the number of // items read has reached this set limit, // then stop reading. @@ -285,7 +317,12 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, } listAccessor.SetResourceVersion(resourceVersion) - return apimeta.SetList(list, allItems) + if err := apimeta.SetList(list, allItems); err != nil { + return err + } + + list.SetContinue("continue-not-supported") + return nil } // multiNamespaceInformer knows how to handle interacting with the underlying informer across multiple namespaces. @@ -293,23 +330,92 @@ type multiNamespaceInformer struct { namespaceToInformer map[string]Informer } +type handlerRegistration struct { + handles map[string]toolscache.ResourceEventHandlerRegistration +} + +// HasSynced asserts that the handler has been called for the full initial state of the informer. +func (h handlerRegistration) HasSynced() bool { + for _, h := range h.handles { + if !h.HasSynced() { + return false + } + } + return true +} + var _ Informer = &multiNamespaceInformer{} -// AddEventHandler adds the handler to each namespaced informer. -func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) { - for _, informer := range i.namespaceToInformer { - informer.AddEventHandler(handler) +// AddEventHandler adds the handler to each informer. +func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) { + handles := handlerRegistration{ + handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)), } + + for ns, informer := range i.namespaceToInformer { + registration, err := informer.AddEventHandler(handler) + if err != nil { + return nil, err + } + handles.handles[ns] = registration + } + + return handles, nil } // AddEventHandlerWithResyncPeriod adds the handler with a resync period to each namespaced informer. -func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) { - for _, informer := range i.namespaceToInformer { - informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod) +func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) { + handles := handlerRegistration{ + handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)), } + + for ns, informer := range i.namespaceToInformer { + registration, err := informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod) + if err != nil { + return nil, err + } + handles.handles[ns] = registration + } + + return handles, nil } -// AddIndexers adds the indexer for each namespaced informer. +// AddEventHandlerWithOptions adds the handler with options to each namespaced informer. +func (i *multiNamespaceInformer) AddEventHandlerWithOptions(handler toolscache.ResourceEventHandler, options toolscache.HandlerOptions) (toolscache.ResourceEventHandlerRegistration, error) { + handles := handlerRegistration{ + handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)), + } + + for ns, informer := range i.namespaceToInformer { + registration, err := informer.AddEventHandlerWithOptions(handler, options) + if err != nil { + return nil, err + } + handles.handles[ns] = registration + } + + return handles, nil +} + +// RemoveEventHandler removes a previously added event handler given by its registration handle. +func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHandlerRegistration) error { + handles, ok := h.(handlerRegistration) + if !ok { + return fmt.Errorf("registration is not a registration returned by multiNamespaceInformer") + } + for ns, informer := range i.namespaceToInformer { + registration, ok := handles.handles[ns] + if !ok { + continue + } + if err := informer.RemoveEventHandler(registration); err != nil { + return err + } + } + return nil +} + +// AddIndexers adds the indexers to each informer. func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error { for _, informer := range i.namespaceToInformer { err := informer.AddIndexers(indexers) @@ -320,11 +426,21 @@ func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error return nil } -// HasSynced checks if each namespaced informer has synced. +// HasSynced checks if each informer has synced. func (i *multiNamespaceInformer) HasSynced() bool { for _, informer := range i.namespaceToInformer { - if ok := informer.HasSynced(); !ok { - return ok + if !informer.HasSynced() { + return false + } + } + return true +} + +// IsStopped checks if each namespaced informer has stopped, returns false if any are still running. +func (i *multiNamespaceInformer) IsStopped() bool { + for _, informer := range i.namespaceToInformer { + if stopped := informer.IsStopped(); !stopped { + return false } } return true diff --git a/pkg/certwatcher/certwatcher.go b/pkg/certwatcher/certwatcher.go index e8e0e17a2b..2362d020b8 100644 --- a/pkg/certwatcher/certwatcher.go +++ b/pkg/certwatcher/certwatcher.go @@ -17,27 +17,46 @@ limitations under the License. package certwatcher import ( + "bytes" "context" "crypto/tls" + "fmt" + "os" "sync" + "time" "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) var log = logf.RuntimeLog.WithName("certwatcher") -// CertWatcher watches certificate and key files for changes. When either file -// changes, it reads and parses both and calls an optional callback with the new -// certificate. +const defaultWatchInterval = 10 * time.Second + +// CertWatcher watches certificate and key files for changes. +// It always returns the cached version, +// but periodically reads and parses certificate and key for changes +// and calls an optional callback with the new certificate. type CertWatcher struct { sync.RWMutex currentCert *tls.Certificate watcher *fsnotify.Watcher + interval time.Duration + log logr.Logger certPath string keyPath string + + cachedKeyPEMBlock []byte + + // callback is a function to be invoked when the certificate changes. + callback func(tls.Certificate) } // New returns a new CertWatcher watching the given certificate and key. @@ -47,6 +66,8 @@ func New(certPath, keyPath string) (*CertWatcher, error) { cw := &CertWatcher{ certPath: certPath, keyPath: keyPath, + interval: defaultWatchInterval, + log: log.WithValues("cert", certPath, "key", keyPath), } // Initial read of certificate and key. @@ -62,6 +83,23 @@ func New(certPath, keyPath string) (*CertWatcher, error) { return cw, nil } +// WithWatchInterval sets the watch interval and returns the CertWatcher pointer +func (cw *CertWatcher) WithWatchInterval(interval time.Duration) *CertWatcher { + cw.interval = interval + return cw +} + +// RegisterCallback registers a callback to be invoked when the certificate changes. +func (cw *CertWatcher) RegisterCallback(callback func(tls.Certificate)) { + cw.Lock() + defer cw.Unlock() + // If the current certificate is not nil, invoke the callback immediately. + if cw.currentCert != nil { + callback(*cw.currentCert) + } + cw.callback = callback +} + // GetCertificate fetches the currently loaded certificate, which may be nil. func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { cw.RLock() @@ -71,22 +109,41 @@ func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, // Start starts the watch on the certificate and key files. func (cw *CertWatcher) Start(ctx context.Context) error { - files := []string{cw.certPath, cw.keyPath} - - for _, f := range files { - if err := cw.watcher.Add(f); err != nil { - return err + files := sets.New(cw.certPath, cw.keyPath) + + { + var watchErr error + if err := wait.PollUntilContextTimeout(ctx, 1*time.Second, 10*time.Second, true, func(ctx context.Context) (done bool, err error) { + for _, f := range files.UnsortedList() { + if err := cw.watcher.Add(f); err != nil { + watchErr = err + return false, nil //nolint:nilerr // We want to keep trying. + } + // We've added the watch, remove it from the set. + files.Delete(f) + } + return true, nil + }); err != nil { + return fmt.Errorf("failed to add watches: %w", kerrors.NewAggregate([]error{err, watchErr})) } } go cw.Watch() - log.Info("Starting certificate watcher") + ticker := time.NewTicker(cw.interval) + defer ticker.Stop() - // Block until the context is done. - <-ctx.Done() - - return cw.watcher.Close() + cw.log.Info("Starting certificate poll+watcher", "interval", cw.interval) + for { + select { + case <-ctx.Done(): + return cw.watcher.Close() + case <-ticker.C: + if err := cw.ReadCertificate(); err != nil { + cw.log.Error(err, "failed read certificate") + } + } + } } // Watch reads events from the watcher's channel and reacts to changes. @@ -100,64 +157,95 @@ func (cw *CertWatcher) Watch() { } cw.handleEvent(event) - case err, ok := <-cw.watcher.Errors: // Channel is closed. if !ok { return } - log.Error(err, "certificate watch error") + cw.log.Error(err, "certificate watch error") } } } +// updateCachedCertificate checks if the new certificate differs from the cache, +// updates it and returns the result if it was updated or not +func (cw *CertWatcher) updateCachedCertificate(cert *tls.Certificate, keyPEMBlock []byte) bool { + cw.Lock() + defer cw.Unlock() + + if cw.currentCert != nil && + bytes.Equal(cw.currentCert.Certificate[0], cert.Certificate[0]) && + bytes.Equal(cw.cachedKeyPEMBlock, keyPEMBlock) { + cw.log.V(7).Info("certificate already cached") + return false + } + cw.currentCert = cert + cw.cachedKeyPEMBlock = keyPEMBlock + return true +} + // ReadCertificate reads the certificate and key files from disk, parses them, -// and updates the current certificate on the watcher. If a callback is set, it +// and updates the current certificate on the watcher if updated. If a callback is set, it // is invoked with the new certificate. func (cw *CertWatcher) ReadCertificate() error { - cert, err := tls.LoadX509KeyPair(cw.certPath, cw.keyPath) + metrics.ReadCertificateTotal.Inc() + certPEMBlock, err := os.ReadFile(cw.certPath) + if err != nil { + metrics.ReadCertificateErrors.Inc() + return err + } + keyPEMBlock, err := os.ReadFile(cw.keyPath) if err != nil { + metrics.ReadCertificateErrors.Inc() return err } - cw.Lock() - cw.currentCert = &cert - cw.Unlock() + cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + metrics.ReadCertificateErrors.Inc() + return err + } - log.Info("Updated current TLS certificate") + if !cw.updateCachedCertificate(&cert, keyPEMBlock) { + return nil + } + cw.log.Info("Updated current TLS certificate") + + // If a callback is registered, invoke it with the new certificate. + cw.RLock() + defer cw.RUnlock() + if cw.callback != nil { + go func() { + cw.callback(cert) + }() + } return nil } func (cw *CertWatcher) handleEvent(event fsnotify.Event) { // Only care about events which may modify the contents of the file. - if !(isWrite(event) || isRemove(event) || isCreate(event)) { - return - } - - log.V(1).Info("certificate event", "event", event) - - // If the file was removed, re-add the watch. - if isRemove(event) { + switch { + case event.Op.Has(fsnotify.Write): + case event.Op.Has(fsnotify.Create): + case event.Op.Has(fsnotify.Chmod), event.Op.Has(fsnotify.Remove): + // If the file was removed or renamed, re-add the watch to the previous name if err := cw.watcher.Add(event.Name); err != nil { - log.Error(err, "error re-watching file") + cw.log.Error(err, "error re-watching file") } + default: + return } + cw.log.V(1).Info("certificate event", "event", event) if err := cw.ReadCertificate(); err != nil { - log.Error(err, "error re-reading certificate") + cw.log.Error(err, "error re-reading certificate") } } -func isWrite(event fsnotify.Event) bool { - return event.Op&fsnotify.Write == fsnotify.Write -} - -func isCreate(event fsnotify.Event) bool { - return event.Op&fsnotify.Create == fsnotify.Create -} - -func isRemove(event fsnotify.Event) bool { - return event.Op&fsnotify.Remove == fsnotify.Remove +// NeedLeaderElection indicates that the cert-manager +// does not need leader election. +func (cw *CertWatcher) NeedLeaderElection() bool { + return false } diff --git a/pkg/certwatcher/certwatcher_suite_test.go b/pkg/certwatcher/certwatcher_suite_test.go index e1e9861ea5..d0d9a72a62 100644 --- a/pkg/certwatcher/certwatcher_suite_test.go +++ b/pkg/certwatcher/certwatcher_suite_test.go @@ -20,9 +20,9 @@ import ( "os" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) @@ -34,16 +34,15 @@ var ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "CertWatcher Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "CertWatcher Suite") } var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) -}, 60) +}) var _ = AfterSuite(func() { - for _, file := range []string{certPath, keyPath} { + for _, file := range []string{certPath, keyPath, certPath + ".new", keyPath + ".new", certPath + ".old", keyPath + ".old"} { _ = os.Remove(file) } -}, 60) +}) diff --git a/pkg/certwatcher/certwatcher_test.go b/pkg/certwatcher/certwatcher_test.go index 287645df92..9737925a6b 100644 --- a/pkg/certwatcher/certwatcher_test.go +++ b/pkg/certwatcher/certwatcher_test.go @@ -20,39 +20,47 @@ import ( "context" "crypto/rand" "crypto/rsa" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "math/big" "net" "os" + "sync/atomic" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus/testutil" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics" + "sigs.k8s.io/controller-runtime/pkg/manager" ) var _ = Describe("CertWatcher", func() { var _ = Describe("certwatcher New", func() { It("should errors without cert/key", func() { _, err := certwatcher.New("", "") - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) }) }) var _ = Describe("certwatcher Start", func() { var ( - ctx context.Context - ctxCancel context.CancelFunc - watcher *certwatcher.CertWatcher + ctxCancel context.CancelFunc + watcher *certwatcher.CertWatcher + startWatcher func(interval time.Duration) (done <-chan struct{}) ) BeforeEach(func() { - ctx, ctxCancel = context.WithCancel(context.Background()) + var ctx context.Context + ctx, ctxCancel = context.WithCancel(context.Background()) //nolint:forbidigo // the watcher outlives the BeforeEach err := writeCerts(certPath, keyPath, "127.0.0.1") - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Eventually(func() error { for _, file := range []string{certPath, keyPath} { @@ -67,47 +75,181 @@ var _ = Describe("CertWatcher", func() { }).Should(Succeed()) watcher, err = certwatcher.New(certPath, keyPath) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + startWatcher = func(interval time.Duration) (done <-chan struct{}) { + doneCh := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(doneCh) + Expect(watcher.WithWatchInterval(interval).Start(ctx)).To(Succeed()) + }() + // wait till we read first cert + Eventually(func() error { + err := watcher.ReadCertificate() + return err + }).Should(Succeed()) + return doneCh + } }) - startWatcher := func() (done <-chan struct{}) { - doneCh := make(chan struct{}) - go func() { - defer GinkgoRecover() - defer close(doneCh) - Expect(watcher.Start(ctx)).To(Succeed()) - }() - // wait till we read first cert - Eventually(func() error { - err := watcher.ReadCertificate() - return err - }).Should(Succeed()) - return doneCh - } + It("should not require LeaderElection", func() { + leaderElectionRunnable, isLeaderElectionRunnable := any(watcher).(manager.LeaderElectionRunnable) + Expect(isLeaderElectionRunnable).To(BeTrue()) + Expect(leaderElectionRunnable.NeedLeaderElection()).To(BeFalse()) + }) It("should read the initial cert/key", func() { - doneCh := startWatcher() + // This test verifies the initial read succeeded. So interval doesn't matter. + doneCh := startWatcher(10 * time.Second) ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) }) It("should reload currentCert when changed", func() { - doneCh := startWatcher() + // This test verifies fsnotify detects the cert change. So interval doesn't matter. + doneCh := startWatcher(10 * time.Second) + called := atomic.Int64{} + watcher.RegisterCallback(func(crt tls.Certificate) { + called.Add(1) + Expect(crt.Certificate).ToNot(BeEmpty()) + }) firstcert, _ := watcher.GetCertificate(nil) err := writeCerts(certPath, keyPath, "192.168.0.1") - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + secondcert, _ := watcher.GetCertificate(nil) + first := firstcert.PrivateKey.(*rsa.PrivateKey) + return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber + }).ShouldNot(BeTrue()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + Expect(called.Load()).To(BeNumerically(">=", 1)) + }) + + It("should reload currentCert when changed with rename", func() { + // This test verifies fsnotify detects the cert change. So interval doesn't matter. + doneCh := startWatcher(10 * time.Second) + called := atomic.Int64{} + watcher.RegisterCallback(func(crt tls.Certificate) { + called.Add(1) + Expect(crt.Certificate).ToNot(BeEmpty()) + }) + + firstcert, _ := watcher.GetCertificate(nil) + + err := writeCerts(certPath+".new", keyPath+".new", "192.168.0.2") + Expect(err).ToNot(HaveOccurred()) + + Expect(os.Link(certPath, certPath+".old")).To(Succeed()) + Expect(os.Rename(certPath+".new", certPath)).To(Succeed()) + + Expect(os.Link(keyPath, keyPath+".old")).To(Succeed()) + Expect(os.Rename(keyPath+".new", keyPath)).To(Succeed()) Eventually(func() bool { secondcert, _ := watcher.GetCertificate(nil) first := firstcert.PrivateKey.(*rsa.PrivateKey) - return first.Equal(secondcert.PrivateKey) + return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber }).ShouldNot(BeTrue()) ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) + Expect(called.Load()).To(BeNumerically(">=", 1)) + }) + + It("should reload currentCert after move out", func() { + // This test verifies poll works, so we'll use 1s as interval (fsnotify doesn't detect this change). + doneCh := startWatcher(1 * time.Second) + called := atomic.Int64{} + watcher.RegisterCallback(func(crt tls.Certificate) { + called.Add(1) + Expect(crt.Certificate).ToNot(BeEmpty()) + }) + + firstcert, _ := watcher.GetCertificate(nil) + + Expect(os.Rename(certPath, certPath+".old")).To(Succeed()) + Expect(os.Rename(keyPath, keyPath+".old")).To(Succeed()) + + err := writeCerts(certPath, keyPath, "192.168.0.3") + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + secondcert, _ := watcher.GetCertificate(nil) + first := firstcert.PrivateKey.(*rsa.PrivateKey) + return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber + }, "10s", "1s").ShouldNot(BeTrue()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + Expect(called.Load()).To(BeNumerically(">=", 1)) + }) + + Context("prometheus metric read_certificate_total", func() { + var readCertificateTotalBefore float64 + var readCertificateErrorsBefore float64 + + BeforeEach(func() { + readCertificateTotalBefore = testutil.ToFloat64(metrics.ReadCertificateTotal) + readCertificateErrorsBefore = testutil.ToFloat64(metrics.ReadCertificateErrors) + }) + + It("should get updated on successful certificate read", func() { + // This test verifies fsnotify, so interval doesn't matter. + doneCh := startWatcher(10 * time.Second) + + Eventually(func() error { + readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) + if readCertificateTotalAfter < readCertificateTotalBefore+1.0 { + return fmt.Errorf("metric read certificate total expected at least: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + } + return nil + }, "4s").Should(Succeed()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + }) + + It("should get updated on read certificate errors", func() { + // This test works with fsnotify, so interval doesn't matter. + doneCh := startWatcher(10 * time.Second) + + Eventually(func() error { + readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) + if readCertificateTotalAfter < readCertificateTotalBefore+1.0 { + return fmt.Errorf("metric read certificate total expected at least: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + } + readCertificateTotalBefore = readCertificateTotalAfter + return nil + }, "4s").Should(Succeed()) + + Expect(os.Remove(keyPath)).To(Succeed()) + + // Note, we are checking two errors here, because os.Remove generates two fsnotify events: Chmod + Remove + Eventually(func() error { + readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) + if readCertificateTotalAfter < readCertificateTotalBefore+2.0 { + return fmt.Errorf("metric read certificate total expected at least: %v and got: %v", readCertificateTotalBefore+2.0, readCertificateTotalAfter) + } + return nil + }, "4s").Should(Succeed()) + Eventually(func() error { + readCertificateErrorsAfter := testutil.ToFloat64(metrics.ReadCertificateErrors) + if readCertificateErrorsAfter < readCertificateErrorsBefore+2.0 { + return fmt.Errorf("metric read certificate errors expected at least: %v and got: %v", readCertificateErrorsBefore+2.0, readCertificateErrorsAfter) + } + return nil + }, "4s").Should(Succeed()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + }) }) }) }) @@ -167,7 +309,7 @@ func writeCerts(certPath, keyPath, ip string) error { return err } - keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) //nolint:gosec + keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return err } diff --git a/pkg/certwatcher/example_test.go b/pkg/certwatcher/example_test.go index 541a73c532..e85b2403cb 100644 --- a/pkg/certwatcher/example_test.go +++ b/pkg/certwatcher/example_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/tls" "net/http" + "time" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/certwatcher" @@ -38,7 +39,7 @@ func Example() { panic(err) } - // Start goroutine with certwatcher running fsnotify against supplied certdir + // Start goroutine with certwatcher running against supplied cert go func() { if err := watcher.Start(ctx); err != nil { panic(err) @@ -56,13 +57,16 @@ func Example() { // Initialize your tls server srv := &http.Server{ - Handler: &sampleServer{}, + Handler: &sampleServer{}, + ReadHeaderTimeout: 5 * time.Second, } // Start goroutine for handling server shutdown. go func() { <-ctx.Done() - if err := srv.Shutdown(context.Background()); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { panic(err) } }() diff --git a/pkg/certwatcher/metrics/metrics.go b/pkg/certwatcher/metrics/metrics.go new file mode 100644 index 0000000000..f128abbcf0 --- /dev/null +++ b/pkg/certwatcher/metrics/metrics.go @@ -0,0 +1,46 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // ReadCertificateTotal is a prometheus counter metrics which holds the total + // number of certificate reads. + ReadCertificateTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "certwatcher_read_certificate_total", + Help: "Total number of certificate reads", + }) + + // ReadCertificateErrors is a prometheus counter metrics which holds the total + // number of errors from certificate read. + ReadCertificateErrors = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "certwatcher_read_certificate_errors_total", + Help: "Total number of certificate read errors", + }) +) + +func init() { + metrics.Registry.MustRegister( + ReadCertificateTotal, + ReadCertificateErrors, + ) +} diff --git a/pkg/client/apiutil/apimachinery.go b/pkg/client/apiutil/apimachinery.go index c92b0eaaec..b132cb2d4d 100644 --- a/pkg/client/apiutil/apimachinery.go +++ b/pkg/client/apiutil/apimachinery.go @@ -20,7 +20,9 @@ limitations under the License. package apiutil import ( + "errors" "fmt" + "net/http" "reflect" "sync" @@ -29,10 +31,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" ) var ( @@ -57,19 +58,37 @@ func AddToProtobufScheme(addToScheme func(*runtime.Scheme) error) error { return addToScheme(protobufScheme) } -// NewDiscoveryRESTMapper constructs a new RESTMapper based on discovery -// information fetched by a new client with the given config. -func NewDiscoveryRESTMapper(c *rest.Config) (meta.RESTMapper, error) { - // Get a mapper - dc, err := discovery.NewDiscoveryClientForConfig(c) +// IsObjectNamespaced returns true if the object is namespace scoped. +// For unstructured objects the gvk is found from the object itself. +func IsObjectNamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper meta.RESTMapper) (bool, error) { + gvk, err := GVKForObject(obj, scheme) if err != nil { - return nil, err + return false, err } - gr, err := restmapper.GetAPIGroupResources(dc) + + return IsGVKNamespaced(gvk, restmapper) +} + +// IsGVKNamespaced returns true if the object having the provided +// GVK is namespace scoped. +func IsGVKNamespaced(gvk schema.GroupVersionKind, restmapper meta.RESTMapper) (bool, error) { + // Fetch the RESTMapping using the complete GVK. If we exclude the Version, the Version set + // will be populated using the cached Group if available. This can lead to failures updating + // the cache with new Versions of CRDs registered at runtime. + restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}, gvk.Version) if err != nil { - return nil, err + return false, fmt.Errorf("failed to get restmapping: %w", err) + } + + scope := restmapping.Scope.Name() + if scope == "" { + return false, errors.New("scope cannot be identified, empty scope returned") + } + + if scope != meta.RESTScopeNameRoot { + return true, nil } - return restmapper.NewDiscoveryRESTMapper(gr), nil + return false, nil } // GVKForObject finds the GroupVersionKind associated with the given object, if there is only a single such GVK. @@ -81,7 +100,7 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi // (unstructured, partial, etc) // check for PartialObjectMetadata, which is analogous to unstructured, but isn't handled by ObjectKinds - _, isPartial := obj.(*metav1.PartialObjectMetadata) //nolint:ifshort + _, isPartial := obj.(*metav1.PartialObjectMetadata) _, isPartialList := obj.(*metav1.PartialObjectMetadataList) if isPartial || isPartialList { // we require that the GVK be populated in order to recognize the object @@ -95,6 +114,7 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi return gvk, nil } + // Use the given scheme to retrieve all the GVKs for the object. gvks, isUnversioned, err := scheme.ObjectKinds(obj) if err != nil { return schema.GroupVersionKind{}, err @@ -103,40 +123,65 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi return schema.GroupVersionKind{}, fmt.Errorf("cannot create group-version-kind for unversioned type %T", obj) } - if len(gvks) < 1 { - return schema.GroupVersionKind{}, fmt.Errorf("no group-version-kinds associated with type %T", obj) - } - if len(gvks) > 1 { - // this should only trigger for things like metav1.XYZ -- - // normal versioned types should be fine + switch { + case len(gvks) < 1: + // If the object has no GVK, the object might not have been registered with the scheme. + // or it's not a valid object. + return schema.GroupVersionKind{}, fmt.Errorf("no GroupVersionKind associated with Go type %T, was the type registered with the Scheme?", obj) + case len(gvks) > 1: + err := fmt.Errorf("multiple GroupVersionKinds associated with Go type %T within the Scheme, this can happen when a type is registered for multiple GVKs at the same time", obj) + + // We've found multiple GVKs for the object. + currentGVK := obj.GetObjectKind().GroupVersionKind() + if !currentGVK.Empty() { + // If the base object has a GVK, check if it's in the list of GVKs before using it. + for _, gvk := range gvks { + if gvk == currentGVK { + return gvk, nil + } + } + + return schema.GroupVersionKind{}, fmt.Errorf( + "%w: the object's supplied GroupVersionKind %q was not found in the Scheme's list; refusing to guess at one: %q", err, currentGVK, gvks) + } + + // This should only trigger for things like metav1.XYZ -- + // normal versioned types should be fine. + // + // See https://github.com/kubernetes-sigs/controller-runtime/issues/362 + // for more information. return schema.GroupVersionKind{}, fmt.Errorf( - "multiple group-version-kinds associated with type %T, refusing to guess at one", obj) + "%w: callers can either fix their type registration to only register it once, or specify the GroupVersionKind to use for object passed in; refusing to guess at one: %q", err, gvks) + default: + // In any other case, we've found a single GVK for the object. + return gvks[0], nil } - return gvks[0], nil } // RESTClientForGVK constructs a new rest.Interface capable of accessing the resource associated // with the given GroupVersionKind. The REST client will be configured to use the negotiated serializer from // baseConfig, if set, otherwise a default serializer will be set. -func RESTClientForGVK(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory) (rest.Interface, error) { - return rest.RESTClientFor(createRestConfig(gvk, isUnstructured, baseConfig, codecs)) -} - -// serializerWithDecodedGVK is a CodecFactory that overrides the DecoderToVersion of a WithoutConversionCodecFactory -// in order to avoid clearing the GVK from the decoded object. -// -// See https://github.com/kubernetes/kubernetes/issues/80609. -type serializerWithDecodedGVK struct { - serializer.WithoutConversionCodecFactory -} - -// DecoderToVersion returns an decoder that does not do conversion. -func (f serializerWithDecodedGVK) DecoderToVersion(serializer runtime.Decoder, _ runtime.GroupVersioner) runtime.Decoder { - return serializer +func RESTClientForGVK( + gvk schema.GroupVersionKind, + forceDisableProtoBuf bool, + isUnstructured bool, + baseConfig *rest.Config, + codecs serializer.CodecFactory, + httpClient *http.Client, +) (rest.Interface, error) { + if httpClient == nil { + return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client") + } + return rest.RESTClientForConfigAndClient(createRestConfig(gvk, forceDisableProtoBuf, isUnstructured, baseConfig, codecs), httpClient) } // createRestConfig copies the base config and updates needed fields for a new rest config. -func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory) *rest.Config { +func createRestConfig(gvk schema.GroupVersionKind, + forceDisableProtoBuf bool, + isUnstructured bool, + baseConfig *rest.Config, + codecs serializer.CodecFactory, +) *rest.Config { gv := gvk.GroupVersion() cfg := rest.CopyConfig(baseConfig) @@ -150,7 +195,7 @@ func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConf cfg.UserAgent = rest.DefaultKubernetesUserAgent() } // TODO(FillZpp): In the long run, we want to check discovery or something to make sure that this is actually true. - if cfg.ContentType == "" && !isUnstructured { + if cfg.ContentType == "" && !forceDisableProtoBuf { protobufSchemeLock.RLock() if protobufScheme.Recognizes(gvk) { cfg.ContentType = runtime.ContentTypeProtobuf @@ -159,9 +204,8 @@ func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConf } if isUnstructured { - // If the object is unstructured, we need to preserve the GVK information. - // Use our own custom serializer. - cfg.NegotiatedSerializer = serializerWithDecodedGVK{serializer.WithoutConversionCodecFactory{CodecFactory: codecs}} + // If the object is unstructured, we use the client-go dynamic serializer. + cfg = dynamic.ConfigFor(cfg) } else { cfg.NegotiatedSerializer = serializerWithTargetZeroingDecode{NegotiatedSerializer: serializer.WithoutConversionCodecFactory{CodecFactory: codecs}} } diff --git a/pkg/client/apiutil/apimachinery_test.go b/pkg/client/apiutil/apimachinery_test.go new file mode 100644 index 0000000000..122c5cc542 --- /dev/null +++ b/pkg/client/apiutil/apimachinery_test.go @@ -0,0 +1,191 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil_test + +import ( + "context" + "strconv" + "testing" + + gmg "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +func TestApiMachinery(t *testing.T) { + for _, aggregatedDiscovery := range []bool{true, false} { + t.Run("aggregatedDiscovery="+strconv.FormatBool(aggregatedDiscovery), func(t *testing.T) { + restCfg := setupEnvtest(t, !aggregatedDiscovery) + + // Details of the GVK registered at initialization. + initialGvk := metav1.GroupVersionKind{ + Group: "crew.example.com", + Version: "v1", + Kind: "Driver", + } + + // A set of GVKs to register at runtime with varying properties. + runtimeGvks := []struct { + name string + gvk metav1.GroupVersionKind + plural string + }{ + { + name: "new Kind and Version added to existing Group", + gvk: metav1.GroupVersionKind{ + Group: "crew.example.com", + Version: "v1alpha1", + Kind: "Passenger", + }, + plural: "passengers", + }, + { + name: "new Kind added to existing Group and Version", + gvk: metav1.GroupVersionKind{ + Group: "crew.example.com", + Version: "v1", + Kind: "Garage", + }, + plural: "garages", + }, + { + name: "new GVK", + gvk: metav1.GroupVersionKind{ + Group: "inventory.example.com", + Version: "v1", + Kind: "Taxi", + }, + plural: "taxis", + }, + } + + t.Run("IsGVKNamespaced should report scope for GVK registered at initialization", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Query the scope of a GVK that was registered at initialization. + scope, err := apiutil.IsGVKNamespaced( + schema.GroupVersionKind(initialGvk), + lazyRestMapper, + ) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(scope).To(gmg.BeTrue()) + }) + + for _, runtimeGvk := range runtimeGvks { + t.Run("IsGVKNamespaced should report scope for "+runtimeGvk.name, func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + c, err := client.New(restCfg, client.Options{Scheme: s}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Run a valid query to initialize cache. + scope, err := apiutil.IsGVKNamespaced( + schema.GroupVersionKind(initialGvk), + lazyRestMapper, + ) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(scope).To(gmg.BeTrue()) + + // Register a new CRD at runtime. + crd := newCRD(t.Context(), g, c, runtimeGvk.gvk.Group, runtimeGvk.gvk.Kind, runtimeGvk.plural) + version := crd.Spec.Versions[0] + version.Name = runtimeGvk.gvk.Version + version.Storage = true + version.Served = true + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{version} + crd.Spec.Scope = apiextensionsv1.NamespaceScoped + + g.Expect(c.Create(t.Context(), crd)).To(gmg.Succeed()) + t.Cleanup(func() { + g.Expect(c.Delete(context.Background(), crd)).To(gmg.Succeed()) //nolint:forbidigo //t.Context is cancelled in t.Cleanup + }) + + // Wait until the CRD is registered. + g.Eventually(func(g gmg.Gomega) { + isRegistered, err := isCrdRegistered(restCfg, runtimeGvk.gvk) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(isRegistered).To(gmg.BeTrue()) + }).Should(gmg.Succeed(), "GVK should be available") + + // Query the scope of the GVK registered at runtime. + scope, err = apiutil.IsGVKNamespaced( + schema.GroupVersionKind(runtimeGvk.gvk), + lazyRestMapper, + ) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(scope).To(gmg.BeTrue()) + }) + } + }) + } +} + +// Check if a slice of APIResource contains a given Kind. +func kindInAPIResources(resources *metav1.APIResourceList, kind string) bool { + for _, res := range resources.APIResources { + if res.Kind == kind { + return true + } + } + return false +} + +// Check if a CRD has registered with the API server using a DiscoveryClient. +func isCrdRegistered(cfg *rest.Config, gvk metav1.GroupVersionKind) (bool, error) { + discHTTP, err := rest.HTTPClientFor(cfg) + if err != nil { + return false, err + } + + discClient, err := discovery.NewDiscoveryClientForConfigAndClient(cfg, discHTTP) + if err != nil { + return false, err + } + + resources, err := discClient.ServerResourcesForGroupVersion(gvk.Group + "/" + gvk.Version) + if err != nil { + return false, err + } + + return kindInAPIResources(resources, gvk.Kind), nil +} diff --git a/pkg/client/apiutil/apiutil_suite_test.go b/pkg/client/apiutil/apiutil_suite_test.go index 757bc42fec..7fe960b917 100644 --- a/pkg/client/apiutil/apiutil_suite_test.go +++ b/pkg/client/apiutil/apiutil_suite_test.go @@ -14,15 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package apiutil_test +package apiutil import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -30,8 +29,7 @@ import ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "API Utilities Test Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "API Utilities Test Suite") } var cfg *rest.Config @@ -41,4 +39,4 @@ var _ = BeforeSuite(func() { // for things that technically need a rest.Config for defaulting, but don't actually use them cfg = &rest.Config{} -}, 60) +}) diff --git a/pkg/client/apiutil/dynamicrestmapper.go b/pkg/client/apiutil/dynamicrestmapper.go deleted file mode 100644 index 56a00371ff..0000000000 --- a/pkg/client/apiutil/dynamicrestmapper.go +++ /dev/null @@ -1,285 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiutil - -import ( - "errors" - "sync" - - "golang.org/x/time/rate" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/discovery" - "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" -) - -// dynamicRESTMapper is a RESTMapper that dynamically discovers resource -// types at runtime. -type dynamicRESTMapper struct { - mu sync.RWMutex // protects the following fields - staticMapper meta.RESTMapper - limiter *rate.Limiter - newMapper func() (meta.RESTMapper, error) - - lazy bool - // Used for lazy init. - initOnce sync.Once -} - -// DynamicRESTMapperOption is a functional option on the dynamicRESTMapper. -type DynamicRESTMapperOption func(*dynamicRESTMapper) error - -// WithLimiter sets the RESTMapper's underlying limiter to lim. -func WithLimiter(lim *rate.Limiter) DynamicRESTMapperOption { - return func(drm *dynamicRESTMapper) error { - drm.limiter = lim - return nil - } -} - -// WithLazyDiscovery prevents the RESTMapper from discovering REST mappings -// until an API call is made. -var WithLazyDiscovery DynamicRESTMapperOption = func(drm *dynamicRESTMapper) error { - drm.lazy = true - return nil -} - -// WithCustomMapper supports setting a custom RESTMapper refresher instead of -// the default method, which uses a discovery client. -// -// This exists mainly for testing, but can be useful if you need tighter control -// over how discovery is performed, which discovery endpoints are queried, etc. -func WithCustomMapper(newMapper func() (meta.RESTMapper, error)) DynamicRESTMapperOption { - return func(drm *dynamicRESTMapper) error { - drm.newMapper = newMapper - return nil - } -} - -// NewDynamicRESTMapper returns a dynamic RESTMapper for cfg. The dynamic -// RESTMapper dynamically discovers resource types at runtime. opts -// configure the RESTMapper. -func NewDynamicRESTMapper(cfg *rest.Config, opts ...DynamicRESTMapperOption) (meta.RESTMapper, error) { - client, err := discovery.NewDiscoveryClientForConfig(cfg) - if err != nil { - return nil, err - } - drm := &dynamicRESTMapper{ - limiter: rate.NewLimiter(rate.Limit(defaultRefillRate), defaultLimitSize), - newMapper: func() (meta.RESTMapper, error) { - groupResources, err := restmapper.GetAPIGroupResources(client) - if err != nil { - return nil, err - } - return restmapper.NewDiscoveryRESTMapper(groupResources), nil - }, - } - for _, opt := range opts { - if err = opt(drm); err != nil { - return nil, err - } - } - if !drm.lazy { - if err := drm.setStaticMapper(); err != nil { - return nil, err - } - } - return drm, nil -} - -var ( - // defaultRefilRate is the default rate at which potential calls are - // added back to the "bucket" of allowed calls. - defaultRefillRate = 5 - // defaultLimitSize is the default starting/max number of potential calls - // per second. Once a call is used, it's added back to the bucket at a rate - // of defaultRefillRate per second. - defaultLimitSize = 5 -) - -// setStaticMapper sets drm's staticMapper by querying its client, regardless -// of reload backoff. -func (drm *dynamicRESTMapper) setStaticMapper() error { - newMapper, err := drm.newMapper() - if err != nil { - return err - } - drm.staticMapper = newMapper - return nil -} - -// init initializes drm only once if drm is lazy. -func (drm *dynamicRESTMapper) init() (err error) { - drm.initOnce.Do(func() { - if drm.lazy { - err = drm.setStaticMapper() - } - }) - return err -} - -// checkAndReload attempts to call the given callback, which is assumed to be dependent -// on the data in the restmapper. -// -// If the callback returns an error that matches the given error, it will attempt to reload -// the RESTMapper's data and re-call the callback once that's occurred. -// If the callback returns any other error, the function will return immediately regardless. -// -// It will take care of ensuring that reloads are rate-limited and that extraneous calls -// aren't made. If a reload would exceed the limiters rate, it returns the error return by -// the callback. -// It's thread-safe, and worries about thread-safety for the callback (so the callback does -// not need to attempt to lock the restmapper). -func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsReload func() error) error { - // first, check the common path -- data is fresh enough - // (use an IIFE for the lock's defer) - err := func() error { - drm.mu.RLock() - defer drm.mu.RUnlock() - - return checkNeedsReload() - }() - - // NB(directxman12): `Is` and `As` have a confusing relationship -- - // `Is` is like `== or does this implement .Is`, whereas `As` says - // `can I type-assert into` - needsReload := errors.As(err, &needsReloadErr) - if !needsReload { - return err - } - - // if the data wasn't fresh, we'll need to try and update it, so grab the lock... - drm.mu.Lock() - defer drm.mu.Unlock() - - // ... and double-check that we didn't reload in the meantime - err = checkNeedsReload() - needsReload = errors.As(err, &needsReloadErr) - if !needsReload { - return err - } - - // we're still stale, so grab a rate-limit token if we can... - if !drm.limiter.Allow() { - // return error from static mapper here, we have refreshed often enough (exceeding rate of provided limiter) - // so that client's can handle this the same way as a "normal" NoResourceMatchError / NoKindMatchError - return err - } - - // ...reload... - if err := drm.setStaticMapper(); err != nil { - return err - } - - // ...and return the results of the closure regardless - return checkNeedsReload() -} - -// TODO: wrap reload errors on NoKindMatchError with go 1.13 errors. - -func (drm *dynamicRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { - if err := drm.init(); err != nil { - return schema.GroupVersionKind{}, err - } - var gvk schema.GroupVersionKind - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { - var err error - gvk, err = drm.staticMapper.KindFor(resource) - return err - }) - return gvk, err -} - -func (drm *dynamicRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { - if err := drm.init(); err != nil { - return nil, err - } - var gvks []schema.GroupVersionKind - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { - var err error - gvks, err = drm.staticMapper.KindsFor(resource) - return err - }) - return gvks, err -} - -func (drm *dynamicRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { - if err := drm.init(); err != nil { - return schema.GroupVersionResource{}, err - } - - var gvr schema.GroupVersionResource - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { - var err error - gvr, err = drm.staticMapper.ResourceFor(input) - return err - }) - return gvr, err -} - -func (drm *dynamicRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { - if err := drm.init(); err != nil { - return nil, err - } - var gvrs []schema.GroupVersionResource - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { - var err error - gvrs, err = drm.staticMapper.ResourcesFor(input) - return err - }) - return gvrs, err -} - -func (drm *dynamicRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { - if err := drm.init(); err != nil { - return nil, err - } - var mapping *meta.RESTMapping - err := drm.checkAndReload(&meta.NoKindMatchError{}, func() error { - var err error - mapping, err = drm.staticMapper.RESTMapping(gk, versions...) - return err - }) - return mapping, err -} - -func (drm *dynamicRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { - if err := drm.init(); err != nil { - return nil, err - } - var mappings []*meta.RESTMapping - err := drm.checkAndReload(&meta.NoKindMatchError{}, func() error { - var err error - mappings, err = drm.staticMapper.RESTMappings(gk, versions...) - return err - }) - return mappings, err -} - -func (drm *dynamicRESTMapper) ResourceSingularizer(resource string) (string, error) { - if err := drm.init(); err != nil { - return "", err - } - var singular string - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { - var err error - singular, err = drm.staticMapper.ResourceSingularizer(resource) - return err - }) - return singular, err -} diff --git a/pkg/client/apiutil/dynamicrestmapper_test.go b/pkg/client/apiutil/dynamicrestmapper_test.go deleted file mode 100644 index 1e294b793d..0000000000 --- a/pkg/client/apiutil/dynamicrestmapper_test.go +++ /dev/null @@ -1,288 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package apiutil_test - -import ( - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/format" - "github.com/onsi/gomega/types" - "golang.org/x/time/rate" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime/schema" - - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" -) - -var ( - targetGVK = schema.GroupVersionKind{Group: "test.kubebuilder.io", Version: "v1beta1", Kind: "SomeCR"} - targetGVR = targetGVK.GroupVersion().WithResource("somecrs") - targetMapping = meta.RESTMapping{Resource: targetGVR, GroupVersionKind: targetGVK, Scope: meta.RESTScopeNamespace} - - secondGVK = schema.GroupVersionKind{Group: "test.kubebuilder.io", Version: "v1beta1", Kind: "OtherCR"} - secondGVR = secondGVK.GroupVersion().WithResource("othercrs") - secondMapping = meta.RESTMapping{Resource: secondGVR, GroupVersionKind: secondGVK, Scope: meta.RESTScopeNamespace} -) - -var _ = Describe("Dynamic REST Mapper", func() { - var mapper meta.RESTMapper - var addToMapper func(*meta.DefaultRESTMapper) - var lim *rate.Limiter - - BeforeEach(func() { - var err error - addToMapper = func(baseMapper *meta.DefaultRESTMapper) { - baseMapper.Add(targetGVK, meta.RESTScopeNamespace) - } - - lim = rate.NewLimiter(rate.Limit(5), 5) - mapper, err = apiutil.NewDynamicRESTMapper(cfg, apiutil.WithLimiter(lim), apiutil.WithCustomMapper(func() (meta.RESTMapper, error) { - baseMapper := meta.NewDefaultRESTMapper(nil) - addToMapper(baseMapper) - - return baseMapper, nil - })) - Expect(err).NotTo(HaveOccurred()) - }) - - var mapperTest = func(callWithTarget func() error, callWithOther func() error) { - It("should read from the cache when possible", func() { - By("reading successfully once when we expect to succeed") - Expect(callWithTarget()).To(Succeed()) - - By("causing requerying to fail, and trying again") - addToMapper = func(_ *meta.DefaultRESTMapper) { - Fail("shouldn't have re-queried") - } - Expect(callWithTarget()).To(Succeed()) - }) - - It("should reload if not present in the cache", func() { - By("reading target successfully once") - Expect(callWithTarget()).To(Succeed()) - - By("reading other not successfully") - count := 0 - addToMapper = func(baseMapper *meta.DefaultRESTMapper) { - count++ - baseMapper.Add(targetGVK, meta.RESTScopeNamespace) - } - Expect(callWithOther()).To(beNoMatchError()) - Expect(count).To(Equal(1), "should reload exactly once") - - By("reading both successfully now") - addToMapper = func(baseMapper *meta.DefaultRESTMapper) { - baseMapper.Add(targetGVK, meta.RESTScopeNamespace) - baseMapper.Add(secondGVK, meta.RESTScopeNamespace) - } - Expect(callWithOther()).To(Succeed()) - Expect(callWithTarget()).To(Succeed()) - }) - - It("should rate-limit then allow more at configured rate", func() { - By("setting a small limit") - *lim = *rate.NewLimiter(rate.Every(100*time.Millisecond), 1) - - By("forcing a reload after changing the mapper") - addToMapper = func(baseMapper *meta.DefaultRESTMapper) { - baseMapper.Add(secondGVK, meta.RESTScopeNamespace) - } - Expect(callWithOther()).To(Succeed()) - - By("calling another time to trigger rate limiting") - addToMapper = func(baseMapper *meta.DefaultRESTMapper) { - baseMapper.Add(targetGVK, meta.RESTScopeNamespace) - } - // if call consistently fails, we are sure, that it was rate-limited, - // otherwise it would have reloaded and succeeded - Consistently(callWithTarget, "90ms", "10ms").Should(beNoMatchError()) - - By("calling until no longer rate-limited") - // once call succeeds, we are sure, that it was no longer rate-limited, - // as it was allowed to reload and found matching kind/resource - Eventually(callWithTarget, "30ms", "10ms").Should(And(Succeed(), Not(beNoMatchError()))) - }) - - It("should avoid reloading twice if two requests for the same thing come in", func() { - count := 0 - // we use sleeps here to simulate two simulataneous requests by slowing things down - addToMapper = func(baseMapper *meta.DefaultRESTMapper) { - count++ - baseMapper.Add(secondGVK, meta.RESTScopeNamespace) - time.Sleep(100 * time.Millisecond) - } - - By("calling two long-running refreshes in parallel and expecting them to succeed") - done := make(chan struct{}) - go func() { - defer GinkgoRecover() - Expect(callWithOther()).To(Succeed()) - close(done) - }() - - Expect(callWithOther()).To(Succeed()) - - // wait till the other goroutine completes to avoid races from a - // new test writing to mapper, and to make sure we read the right - // count - <-done - - By("ensuring that it was only refreshed once") - Expect(count).To(Equal(1)) - }) - } - - PIt("should lazily initialize if the lazy option is used", func() { - - }) - - Describe("KindFor", func() { - mapperTest(func() error { - gvk, err := mapper.KindFor(targetGVR) - if err == nil { - Expect(gvk).To(Equal(targetGVK)) - } - return err - }, func() error { - gvk, err := mapper.KindFor(secondGVR) - if err == nil { - Expect(gvk).To(Equal(secondGVK)) - } - return err - }) - }) - - Describe("KindsFor", func() { - mapperTest(func() error { - gvk, err := mapper.KindsFor(targetGVR) - if err == nil { - Expect(gvk).To(Equal([]schema.GroupVersionKind{targetGVK})) - } - return err - }, func() error { - gvk, err := mapper.KindsFor(secondGVR) - if err == nil { - Expect(gvk).To(Equal([]schema.GroupVersionKind{secondGVK})) - } - return err - }) - }) - - Describe("ResourceFor", func() { - mapperTest(func() error { - gvk, err := mapper.ResourceFor(targetGVR) - if err == nil { - Expect(gvk).To(Equal(targetGVR)) - } - return err - }, func() error { - gvk, err := mapper.ResourceFor(secondGVR) - if err == nil { - Expect(gvk).To(Equal(secondGVR)) - } - return err - }) - }) - - Describe("ResourcesFor", func() { - mapperTest(func() error { - gvk, err := mapper.ResourcesFor(targetGVR) - if err == nil { - Expect(gvk).To(Equal([]schema.GroupVersionResource{targetGVR})) - } - return err - }, func() error { - gvk, err := mapper.ResourcesFor(secondGVR) - if err == nil { - Expect(gvk).To(Equal([]schema.GroupVersionResource{secondGVR})) - } - return err - }) - }) - - Describe("RESTMappingFor", func() { - mapperTest(func() error { - gvk, err := mapper.RESTMapping(targetGVK.GroupKind(), targetGVK.Version) - if err == nil { - Expect(gvk).To(Equal(&targetMapping)) - } - return err - }, func() error { - gvk, err := mapper.RESTMapping(secondGVK.GroupKind(), targetGVK.Version) - if err == nil { - Expect(gvk).To(Equal(&secondMapping)) - } - return err - }) - }) - - Describe("RESTMappingsFor", func() { - mapperTest(func() error { - gvk, err := mapper.RESTMappings(targetGVK.GroupKind(), targetGVK.Version) - if err == nil { - Expect(gvk).To(Equal([]*meta.RESTMapping{&targetMapping})) - } - return err - }, func() error { - gvk, err := mapper.RESTMappings(secondGVK.GroupKind(), targetGVK.Version) - if err == nil { - Expect(gvk).To(Equal([]*meta.RESTMapping{&secondMapping})) - } - return err - }) - }) - - Describe("ResourceSingularizer", func() { - mapperTest(func() error { - gvk, err := mapper.ResourceSingularizer(targetGVR.Resource) - if err == nil { - Expect(gvk).To(Equal(targetGVR.Resource[:len(targetGVR.Resource)-1])) - } - return err - }, func() error { - gvk, err := mapper.ResourceSingularizer(secondGVR.Resource) - if err == nil { - Expect(gvk).To(Equal(secondGVR.Resource[:len(secondGVR.Resource)-1])) - } - return err - }) - }) -}) - -func beNoMatchError() types.GomegaMatcher { - return noMatchErrorMatcher{} -} - -type noMatchErrorMatcher struct{} - -func (k noMatchErrorMatcher) Match(actual interface{}) (success bool, err error) { - actualErr, actualOk := actual.(error) - if !actualOk { - return false, nil - } - - return meta.IsNoMatchError(actualErr), nil -} - -func (k noMatchErrorMatcher) FailureMessage(actual interface{}) (message string) { - return format.Message(actual, "to be a NoMatchError") -} -func (k noMatchErrorMatcher) NegatedFailureMessage(actual interface{}) (message string) { - return format.Message(actual, "not to be a NoMatchError") -} diff --git a/pkg/client/apiutil/errors.go b/pkg/client/apiutil/errors.go new file mode 100644 index 0000000000..c216c49d2a --- /dev/null +++ b/pkg/client/apiutil/errors.go @@ -0,0 +1,54 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "fmt" + "sort" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// ErrResourceDiscoveryFailed is returned if the RESTMapper cannot discover supported resources for some GroupVersions. +// It wraps the errors encountered, except "NotFound" errors are replaced with meta.NoResourceMatchError, for +// backwards compatibility with code that uses meta.IsNoMatchError() to check for unsupported APIs. +type ErrResourceDiscoveryFailed map[schema.GroupVersion]error + +// Error implements the error interface. +func (e *ErrResourceDiscoveryFailed) Error() string { + subErrors := []string{} + for k, v := range *e { + subErrors = append(subErrors, fmt.Sprintf("%s: %v", k, v)) + } + sort.Strings(subErrors) + return fmt.Sprintf("unable to retrieve the complete list of server APIs: %s", strings.Join(subErrors, ", ")) +} + +func (e *ErrResourceDiscoveryFailed) Unwrap() []error { + subErrors := []error{} + for gv, err := range *e { + if apierrors.IsNotFound(err) { + err = &meta.NoResourceMatchError{PartialResource: gv.WithResource("")} + } + subErrors = append(subErrors, err) + } + return subErrors +} diff --git a/pkg/client/apiutil/restmapper.go b/pkg/client/apiutil/restmapper.go new file mode 100644 index 0000000000..7a7a0d1145 --- /dev/null +++ b/pkg/client/apiutil/restmapper.go @@ -0,0 +1,372 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "fmt" + "net/http" + "sync" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/utils/ptr" +) + +// NewDynamicRESTMapper returns a dynamic RESTMapper for cfg. The dynamic +// RESTMapper dynamically discovers resource types at runtime. +func NewDynamicRESTMapper(cfg *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + if httpClient == nil { + return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client") + } + + client, err := discovery.NewDiscoveryClientForConfigAndClient(cfg, httpClient) + if err != nil { + return nil, err + } + + return &mapper{ + mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), + client: client, + knownGroups: map[string]*restmapper.APIGroupResources{}, + apiGroups: map[string]*metav1.APIGroup{}, + }, nil +} + +// mapper is a RESTMapper that will lazily query the provided +// client for discovery information to do REST mappings. +type mapper struct { + mapper meta.RESTMapper + client discovery.AggregatedDiscoveryInterface + knownGroups map[string]*restmapper.APIGroupResources + apiGroups map[string]*metav1.APIGroup + + initialDiscoveryDone bool + + // mutex to provide thread-safe mapper reloading. + // It protects all fields in the mapper as well as methods + // that have the `Locked` suffix. + mu sync.RWMutex +} + +// KindFor implements Mapper.KindFor. +func (m *mapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { + res, err := m.getMapper().KindFor(resource) + if meta.IsNoMatchError(err) { + if err := m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil { + return schema.GroupVersionKind{}, err + } + res, err = m.getMapper().KindFor(resource) + } + + return res, err +} + +// KindsFor implements Mapper.KindsFor. +func (m *mapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { + res, err := m.getMapper().KindsFor(resource) + if meta.IsNoMatchError(err) { + if err := m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil { + return nil, err + } + res, err = m.getMapper().KindsFor(resource) + } + + return res, err +} + +// ResourceFor implements Mapper.ResourceFor. +func (m *mapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { + res, err := m.getMapper().ResourceFor(input) + if meta.IsNoMatchError(err) { + if err := m.addKnownGroupAndReload(input.Group, input.Version); err != nil { + return schema.GroupVersionResource{}, err + } + res, err = m.getMapper().ResourceFor(input) + } + + return res, err +} + +// ResourcesFor implements Mapper.ResourcesFor. +func (m *mapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { + res, err := m.getMapper().ResourcesFor(input) + if meta.IsNoMatchError(err) { + if err := m.addKnownGroupAndReload(input.Group, input.Version); err != nil { + return nil, err + } + res, err = m.getMapper().ResourcesFor(input) + } + + return res, err +} + +// RESTMapping implements Mapper.RESTMapping. +func (m *mapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + res, err := m.getMapper().RESTMapping(gk, versions...) + if meta.IsNoMatchError(err) { + if err := m.addKnownGroupAndReload(gk.Group, versions...); err != nil { + return nil, err + } + res, err = m.getMapper().RESTMapping(gk, versions...) + } + + return res, err +} + +// RESTMappings implements Mapper.RESTMappings. +func (m *mapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { + res, err := m.getMapper().RESTMappings(gk, versions...) + if meta.IsNoMatchError(err) { + if err := m.addKnownGroupAndReload(gk.Group, versions...); err != nil { + return nil, err + } + res, err = m.getMapper().RESTMappings(gk, versions...) + } + + return res, err +} + +// ResourceSingularizer implements Mapper.ResourceSingularizer. +func (m *mapper) ResourceSingularizer(resource string) (string, error) { + return m.getMapper().ResourceSingularizer(resource) +} + +func (m *mapper) getMapper() meta.RESTMapper { + m.mu.RLock() + defer m.mu.RUnlock() + return m.mapper +} + +// addKnownGroupAndReload reloads the mapper with updated information about missing API group. +// versions can be specified for partial updates, for instance for v1beta1 version only. +func (m *mapper) addKnownGroupAndReload(groupName string, versions ...string) error { + // versions will here be [""] if the forwarded Version value of + // GroupVersionResource (in calling method) was not specified. + if len(versions) == 1 && versions[0] == "" { + versions = nil + } + + m.mu.Lock() + defer m.mu.Unlock() + // If no specific versions are set by user, we will scan all available ones for the API group. + // This operation requires 2 requests: /api and /apis, but only once. For all subsequent calls + // this data will be taken from cache. + // + // We always run this once, because if the server supports aggregated discovery, this will + // load everything with two api calls which we assume is overall cheaper. + if len(versions) == 0 || !m.initialDiscoveryDone { + apiGroup, didAggregatedDiscovery, err := m.findAPIGroupByNameAndMaybeAggregatedDiscoveryLocked(groupName) + if err != nil { + return err + } + if apiGroup != nil && len(versions) == 0 { + for _, version := range apiGroup.Versions { + versions = append(versions, version.Version) + } + } + + // No need to do anything further if aggregatedDiscovery is supported and we did a lookup + if didAggregatedDiscovery { + failedGroups := make(map[schema.GroupVersion]error) + for _, version := range versions { + if m.knownGroups[groupName] == nil || m.knownGroups[groupName].VersionedResources[version] == nil { + failedGroups[schema.GroupVersion{Group: groupName, Version: version}] = &meta.NoResourceMatchError{ + PartialResource: schema.GroupVersionResource{ + Group: groupName, + Version: version, + }} + } + } + if len(failedGroups) > 0 { + return ptr.To(ErrResourceDiscoveryFailed(failedGroups)) + } + return nil + } + } + + // Update information for group resources about versioned resources. + // The number of API calls is equal to the number of versions: /apis//. + // If we encounter a missing API version (NotFound error), we will remove the group from + // the m.apiGroups and m.knownGroups caches. + // If this happens, in the next call the group will be added back to apiGroups + // and only the existing versions will be loaded in knownGroups. + groupVersionResources, err := m.fetchGroupVersionResourcesLocked(groupName, versions...) + if err != nil { + return fmt.Errorf("failed to get API group resources: %w", err) + } + + m.addGroupVersionResourcesToCacheAndReloadLocked(groupVersionResources) + return nil +} + +// addGroupVersionResourcesToCacheAndReloadLocked does what the name suggests. The mutex must be held when +// calling it. +func (m *mapper) addGroupVersionResourcesToCacheAndReloadLocked(gvr map[schema.GroupVersion]*metav1.APIResourceList) { + // Update information for group resources about the API group by adding new versions. + // Ignore the versions that are already registered + for groupVersion, resources := range gvr { + var groupResources *restmapper.APIGroupResources + if _, ok := m.knownGroups[groupVersion.Group]; ok { + groupResources = m.knownGroups[groupVersion.Group] + } else { + groupResources = &restmapper.APIGroupResources{ + Group: metav1.APIGroup{Name: groupVersion.Group}, + VersionedResources: make(map[string][]metav1.APIResource), + } + } + + version := groupVersion.Version + + groupResources.VersionedResources[version] = resources.APIResources + found := false + for _, v := range groupResources.Group.Versions { + if v.Version == version { + found = true + break + } + } + + if !found { + gv := metav1.GroupVersionForDiscovery{ + GroupVersion: metav1.GroupVersion{Group: groupVersion.Group, Version: version}.String(), + Version: version, + } + + // Prepend if preferred version, else append. The upstream DiscoveryRestMappper assumes + // the first version is the preferred one: https://github.com/kubernetes/kubernetes/blob/ef54ac803b712137871c1a1f8d635d50e69ffa6c/staging/src/k8s.io/apimachinery/pkg/api/meta/restmapper.go#L458-L461 + if group, ok := m.apiGroups[groupVersion.Group]; ok && group.PreferredVersion.Version == version { + groupResources.Group.Versions = append([]metav1.GroupVersionForDiscovery{gv}, groupResources.Group.Versions...) + } else { + groupResources.Group.Versions = append(groupResources.Group.Versions, gv) + } + } + + // Update data in the cache. + m.knownGroups[groupVersion.Group] = groupResources + } + + // Finally, reload the mapper. + updatedGroupResources := make([]*restmapper.APIGroupResources, 0, len(m.knownGroups)) + for _, agr := range m.knownGroups { + updatedGroupResources = append(updatedGroupResources, agr) + } + + m.mapper = restmapper.NewDiscoveryRESTMapper(updatedGroupResources) +} + +// findAPIGroupByNameAndMaybeAggregatedDiscoveryLocked tries to find the passed apiGroup. +// If the server supports aggregated discovery, it will always perform that. +func (m *mapper) findAPIGroupByNameAndMaybeAggregatedDiscoveryLocked(groupName string) (_ *metav1.APIGroup, didAggregatedDiscovery bool, _ error) { + // Looking in the cache first + group, ok := m.apiGroups[groupName] + if ok { + return group, false, nil + } + + // Update the cache if nothing was found. + apiGroups, maybeResources, _, err := m.client.GroupsAndMaybeResources() + if err != nil { + return nil, false, fmt.Errorf("failed to get server groups: %w", err) + } + if len(apiGroups.Groups) == 0 { + return nil, false, fmt.Errorf("received an empty API groups list") + } + + m.initialDiscoveryDone = true + for i := range apiGroups.Groups { + group := &apiGroups.Groups[i] + m.apiGroups[group.Name] = group + } + if len(maybeResources) > 0 { + didAggregatedDiscovery = true + m.addGroupVersionResourcesToCacheAndReloadLocked(maybeResources) + } + + // Looking in the cache again. + // Don't return an error here if the API group is not present. + // The reloaded RESTMapper will take care of returning a NoMatchError. + return m.apiGroups[groupName], didAggregatedDiscovery, nil +} + +// fetchGroupVersionResourcesLocked fetches the resources for the specified group and its versions. +// This method might modify the cache so it needs to be called under the lock. +func (m *mapper) fetchGroupVersionResourcesLocked(groupName string, versions ...string) (map[schema.GroupVersion]*metav1.APIResourceList, error) { + groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) + failedGroups := make(map[schema.GroupVersion]error) + + for _, version := range versions { + groupVersion := schema.GroupVersion{Group: groupName, Version: version} + + apiResourceList, err := m.client.ServerResourcesForGroupVersion(groupVersion.String()) + if apierrors.IsNotFound(err) { + // If the version is not found, we remove the group from the cache + // so it gets refreshed on the next call. + if m.isAPIGroupCachedLocked(groupVersion) { + delete(m.apiGroups, groupName) + } + if m.isGroupVersionCachedLocked(groupVersion) { + delete(m.knownGroups, groupName) + } + continue + } else if err != nil { + failedGroups[groupVersion] = err + } + + if apiResourceList != nil { + // even in case of error, some fallback might have been returned. + groupVersionResources[groupVersion] = apiResourceList + } + } + + if len(failedGroups) > 0 { + err := ErrResourceDiscoveryFailed(failedGroups) + return nil, &err + } + + return groupVersionResources, nil +} + +// isGroupVersionCachedLocked checks if a version for a group is cached in the known groups cache. +func (m *mapper) isGroupVersionCachedLocked(gv schema.GroupVersion) bool { + if cachedGroup, ok := m.knownGroups[gv.Group]; ok { + _, cached := cachedGroup.VersionedResources[gv.Version] + return cached + } + + return false +} + +// isAPIGroupCachedLocked checks if a version for a group is cached in the api groups cache. +func (m *mapper) isAPIGroupCachedLocked(gv schema.GroupVersion) bool { + cachedGroup, ok := m.apiGroups[gv.Group] + if !ok { + return false + } + + for _, version := range cachedGroup.Versions { + if version.Version == gv.Version { + return true + } + } + + return false +} diff --git a/pkg/client/apiutil/restmapper_test.go b/pkg/client/apiutil/restmapper_test.go new file mode 100644 index 0000000000..51807f12de --- /dev/null +++ b/pkg/client/apiutil/restmapper_test.go @@ -0,0 +1,833 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil_test + +import ( + "context" + "fmt" + "net/http" + "strconv" + "sync" + "testing" + + _ "github.com/onsi/ginkgo/v2" + gmg "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + gomegatypes "github.com/onsi/gomega/types" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// countingRoundTripper is used to count HTTP requests. +type countingRoundTripper struct { + roundTripper http.RoundTripper + requestCount int +} + +func newCountingRoundTripper(rt http.RoundTripper) *countingRoundTripper { + return &countingRoundTripper{roundTripper: rt} +} + +// RoundTrip implements http.RoundTripper.RoundTrip that additionally counts requests. +func (crt *countingRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + crt.requestCount++ + + return crt.roundTripper.RoundTrip(r) +} + +// GetRequestCount returns how many requests have been made. +func (crt *countingRoundTripper) GetRequestCount() int { + return crt.requestCount +} + +// Reset sets the counter to 0. +func (crt *countingRoundTripper) Reset() { + crt.requestCount = 0 +} + +func setupEnvtest(t *testing.T, disableAggregatedDiscovery bool) *rest.Config { + t.Log("Setup envtest") + + g := gmg.NewWithT(t) + testEnv := &envtest.Environment{ + CRDDirectoryPaths: []string{"testdata"}, + } + if disableAggregatedDiscovery { + testEnv.DownloadBinaryAssets = true + testEnv.DownloadBinaryAssetsVersion = "v1.28.0" + binaryAssetsDirectory, err := envtest.SetupEnvtestDefaultBinaryAssetsDirectory() + g.Expect(err).ToNot(gmg.HaveOccurred()) + testEnv.BinaryAssetsDirectory = binaryAssetsDirectory + testEnv.ControlPlane.GetAPIServer().Configure().Append("feature-gates", "AggregatedDiscoveryEndpoint=false") + } + + cfg, err := testEnv.Start() + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(cfg).NotTo(gmg.BeNil()) + + t.Cleanup(func() { + t.Log("Stop envtest") + g.Expect(testEnv.Stop()).To(gmg.Succeed()) + }) + + return cfg +} + +func TestLazyRestMapperProvider(t *testing.T) { + for _, aggregatedDiscovery := range []bool{true, false} { + t.Run("aggregatedDiscovery="+strconv.FormatBool(aggregatedDiscovery), func(t *testing.T) { + restCfg := setupEnvtest(t, !aggregatedDiscovery) + + t.Run("LazyRESTMapper should fetch data based on the request", func(t *testing.T) { + g := gmg.NewWithT(t) + + // For each new group it performs just one request to the API server: + // GET https://host/apis// + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + mappings, err := lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mappings).To(gmg.HaveLen(1)) + g.Expect(mappings[0].GroupVersionKind.Kind).To(gmg.Equal("pod")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "ingresses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Kind).To(gmg.Equal("Ingress")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "tokenreviews"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kinds).To(gmg.HaveLen(1)) + g.Expect(kinds[0].Kind).To(gmg.Equal("TokenReview")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + resource, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "priorityclasses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resource.Resource).To(gmg.Equal("priorityclasses")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + resources, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "poddisruptionbudgets"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resources).To(gmg.HaveLen(1)) + g.Expect(resources[0].Resource).To(gmg.Equal("poddisruptionbudgets")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should cache fetched data and doesn't perform any additional requests", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // Data taken from cache - there are no more additional requests. + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + kind, err := lazyRestMapper.KindFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Kind).To(gmg.Equal("Deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + resource, err := lazyRestMapper.ResourceFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resource.Resource).To(gmg.Equal("deployments")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should work correctly with empty versions list", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // crew.example.com has 2 versions: v1 and v2 + + // If no versions were provided by user, we fetch all of them. + // Here we expect 4 calls. + // To initialize: + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each version it performs one request to the API server: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // All subsequent calls won't send requests to the server. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should work correctly with multiple API group versions", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // We explicitly ask for 2 versions: v1 and v2. + // For each version it performs one request to the API server: + // #1: GET https://host/apis/crew.example.com/v1 + // #2: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // All subsequent calls won't send requests to the server as everything is stored in the cache. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should work correctly with different API group versions", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Now we want resources for crew.example.com/v1 version only. + // Here we expect 1 call: + // #1: GET https://host/apis/crew.example.com/v1 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // Get additional resources from v2. + // It sends another request: + // #2: GET https://host/apis/crew.example.com/v2 + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // No subsequent calls require additional API requests. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should return an error if the group doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // After initialization for each invalid group the mapper performs just 1 request to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // A version is specified but the group doesn't exist. + // For each group, we expect 1 call to the version-specific discovery endpoint: + // #1: GET https://host/apis// + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "INVALID1"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "INVALID2"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "INVALID3", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "INVALID4", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "INVALID5", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "INVALID6", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + + // No version is specified but the group doesn't exist. + // For each group, we expect 2 calls to discover all group versions: + // #1: GET https://host/api + // #2: GET https://host/apis + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "INVALID7"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(7)) + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "INVALID8"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(9)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "INVALID9", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(11)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "INVALID10", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(13)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "INVALID11", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(15)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "INVALID12", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(17)) + }) + + t.Run("LazyRESTMapper should return an error if a resource doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // For each invalid resource the mapper performs just 1 request to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "INVALID"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "INVALID"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + }) + + t.Run("LazyRESTMapper should return an error if the version doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // After initialization, for each invalid resource mapper performs 1 requests to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "INVALID") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "INVALID") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "INVALID", Resource: "ingresses"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "INVALID", Resource: "tokenreviews"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "INVALID", Resource: "priorityclasses"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "INVALID", Resource: "poddisruptionbudgets"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + }) + + t.Run("LazyRESTMapper should work correctly if the version isn't specified", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Resource: "ingress"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Version).ToNot(gmg.BeEmpty()) + + kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Resource: "tokenreviews"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kinds).ToNot(gmg.BeEmpty()) + g.Expect(kinds[0].Version).ToNot(gmg.BeEmpty()) + + resorce, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Resource: "priorityclasses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resorce.Version).ToNot(gmg.BeEmpty()) + + resorces, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Resource: "poddisruptionbudgets"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kinds).ToNot(gmg.BeEmpty()) + g.Expect(resorces[0].Version).ToNot(gmg.BeEmpty()) + }) + + t.Run("LazyRESTMapper can fetch CRDs if they were created at runtime", func(t *testing.T) { + g := gmg.NewWithT(t) + + // To fetch all versions mapper does 2 requests: + // GET https://host/api + // GET https://host/apis + // Then, for each version it performs just one request to the API server as usual: + // GET https://host/apis// + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each currently registered version: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + c, err := client.New(restCfg, client.Options{Scheme: s}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Register another CRD in runtime - "riders.crew.example.com". + createNewCRD(t.Context(), g, c, "crew.example.com", "Rider", "riders") + + // Wait a bit until the CRD is registered. + g.Eventually(func() error { + _, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) + return err + }).Should(gmg.Succeed()) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each currently registered version: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("rider")) + }) + + t.Run("LazyRESTMapper should invalidate the group cache if a version is not found", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + c, err := client.New(restCfg, client.Options{Scheme: s}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Register a new CRD ina new group to avoid collisions when deleting versions - "taxis.inventory.example.com". + group := "inventory.example.com" + kind := "Taxi" + plural := "taxis" + crdName := plural + "." + group + // Create a CRD with two versions: v1alpha1 and v1 where both are served and + // v1 is the storage version so we can easily remove v1alpha1 later. + crd := newCRD(t.Context(), g, c, group, kind, plural) + v1alpha1 := crd.Spec.Versions[0] + v1alpha1.Name = "v1alpha1" + v1alpha1.Storage = false + v1alpha1.Served = true + v1 := crd.Spec.Versions[0] + v1.Name = "v1" + v1.Storage = true + v1.Served = true + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{v1alpha1, v1} + g.Expect(c.Create(t.Context(), crd)).To(gmg.Succeed()) + t.Cleanup(func() { + g.Expect(c.Delete(context.Background(), crd)).To(gmg.Succeed()) //nolint:forbidigo //t.Context is cancelled in t.Cleanup + }) + + // Wait until the CRD is registered. + discHTTP, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + discClient, err := discovery.NewDiscoveryClientForConfigAndClient(restCfg, discHTTP) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Eventually(func(g gmg.Gomega) { + _, err = discClient.ServerResourcesForGroupVersion(group + "/v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + }).Should(gmg.Succeed(), "v1 should be available") + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for all available versions: + // #3: GET https://host/apis/inventory.example.com/v1alpha1 + // #4: GET https://host/apis/inventory.example.com/v1 + // This should fill the cache for apiGroups and versions. + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal(kind)) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() // We reset the counter to check how many additional requests are made later. + + // At this point v1alpha1 should be cached + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}, "v1alpha1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // We update the CRD to only have v1 version. + g.Expect(c.Get(t.Context(), types.NamespacedName{Name: crdName}, crd)).To(gmg.Succeed()) + for _, version := range crd.Spec.Versions { + if version.Name == "v1" { + v1 = version + break + } + } + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{v1} + g.Expect(c.Update(t.Context(), crd)).To(gmg.Succeed()) + + // We wait until v1alpha1 is not available anymore. + g.Eventually(func(g gmg.Gomega) { + _, err = discClient.ServerResourcesForGroupVersion(group + "/v1alpha1") + g.Expect(apierrors.IsNotFound(err)).To(gmg.BeTrue(), "v1alpha1 should not be available anymore") + }).Should(gmg.Succeed()) + + // Although v1alpha1 is not available anymore, the cache is not invalidated yet so it should return a mapping. + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}, "v1alpha1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // We request Limo, which is not in the mapper because it doesn't exist. + // This will trigger a reload of the lazy mapper cache. + // Reloading the cache will read v2 again and since it's not available anymore, it should invalidate the cache. + // #1: GET https://host/apis/inventory.example.com/v1alpha1 + // #2: GET https://host/apis/inventory.example.com/v1 + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: "Limo"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + crt.Reset() + + // Now we request v1alpha1 again and it should return an error since the cache was invalidated. + // #1: GET https://host/apis/inventory.example.com/v1alpha1 + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}, "v1alpha1") + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + // Verify that when requesting the mapping without a version, it doesn't error + // and it returns v1. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.Resource.Version).To(gmg.Equal("v1")) + }) + + t.Run("Restmapper should consistently return the preferred version", func(t *testing.T) { + g := gmg.NewWithT(t) + + wg := sync.WaitGroup{} + wg.Add(50) + for i := 0; i < 50; i++ { + go func() { + defer wg.Done() + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + mapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + mapping, err := mapper.RESTMapping(schema.GroupKind{ + Group: "crew.example.com", + Kind: "Driver", + }) + g.Expect(err).NotTo(gmg.HaveOccurred()) + // APIServer seems to have a heuristic to prefer the higher + // version number. + g.Expect(mapping.GroupVersionKind.Version).To(gmg.Equal("v2")) + }() + } + wg.Wait() + }) + }) + } +} + +// createNewCRD creates a new CRD with the given group, kind, and plural and returns it. +func createNewCRD(ctx context.Context, g gmg.Gomega, c client.Client, group, kind, plural string) *apiextensionsv1.CustomResourceDefinition { + newCRD := newCRD(ctx, g, c, group, kind, plural) + g.Expect(c.Create(ctx, newCRD)).To(gmg.Succeed()) + + return newCRD +} + +// newCRD returns a new CRD with the given group, kind, and plural. +func newCRD(ctx context.Context, g gmg.Gomega, c client.Client, group, kind, plural string) *apiextensionsv1.CustomResourceDefinition { + crd := &apiextensionsv1.CustomResourceDefinition{} + err := c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(crd.Spec.Names.Kind).To(gmg.Equal("Driver")) + + newCRD := &apiextensionsv1.CustomResourceDefinition{} + crd.DeepCopyInto(newCRD) + newCRD.Spec.Group = group + newCRD.Name = plural + "." + group + newCRD.Spec.Names = apiextensionsv1.CustomResourceDefinitionNames{ + Kind: kind, + Plural: plural, + } + newCRD.ResourceVersion = "" + + return newCRD +} + +func beNoMatchError() gomegatypes.GomegaMatcher { + return &errorMatcher{ + checkFunc: meta.IsNoMatchError, + message: "NoMatch", + } +} + +type errorMatcher struct { + checkFunc func(error) bool + message string +} + +func (e *errorMatcher) Match(actual interface{}) (success bool, err error) { + if actual == nil { + return false, nil + } + + actualErr, actualOk := actual.(error) + if !actualOk { + return false, fmt.Errorf("expected an error-type. got:\n%s", format.Object(actual, 1)) + } + + return e.checkFunc(actualErr), nil +} + +func (e *errorMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("to be %s error", e.message)) +} + +func (e *errorMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("not to be %s error", e.message)) +} diff --git a/pkg/client/apiutil/restmapper_wb_test.go b/pkg/client/apiutil/restmapper_wb_test.go new file mode 100644 index 0000000000..73c4236724 --- /dev/null +++ b/pkg/client/apiutil/restmapper_wb_test.go @@ -0,0 +1,214 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "testing" + + gmg "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/restmapper" +) + +func TestLazyRestMapper_fetchGroupVersionResourcesLocked_CacheInvalidation(t *testing.T) { + tests := []struct { + name string + groupName string + versions []string + cachedAPIGroups, expectedAPIGroups map[string]*metav1.APIGroup + cachedKnownGroups, expectedKnownGroups map[string]*restmapper.APIGroupResources + }{ + { + name: "Not found version for cached groupVersion in apiGroups and knownGroups", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v1", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v1": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{}, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{}, + }, + { + name: "Not found version for cached groupVersion only in apiGroups", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v1", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{}, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + }, + { + name: "Not found version for cached groupVersion only in knownGroups", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v2": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{}, + }, + { + name: "Not found version for non cached groupVersion", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gmg.NewWithT(t) + m := &mapper{ + mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), + client: &fakeAggregatedDiscoveryClient{DiscoveryInterface: fake.NewSimpleClientset().Discovery()}, + apiGroups: tt.cachedAPIGroups, + knownGroups: tt.cachedKnownGroups, + } + _, err := m.fetchGroupVersionResourcesLocked(tt.groupName, tt.versions...) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(m.apiGroups).To(gmg.BeComparableTo(tt.expectedAPIGroups)) + g.Expect(m.knownGroups).To(gmg.BeComparableTo(tt.expectedKnownGroups)) + }) + } +} + +type fakeAggregatedDiscoveryClient struct { + discovery.DiscoveryInterface +} + +func (f *fakeAggregatedDiscoveryClient) GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error, error) { + groupList, err := f.DiscoveryInterface.ServerGroups() + return groupList, nil, nil, err +} diff --git a/pkg/client/apiutil/testdata/crd.yaml b/pkg/client/apiutil/testdata/crd.yaml new file mode 100644 index 0000000000..5bb2d73f69 --- /dev/null +++ b/pkg/client/apiutil/testdata/crd.yaml @@ -0,0 +1,62 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: drivers.crew.example.com +spec: + group: crew.example.com + names: + kind: Driver + plural: drivers + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object + - name: v2 + served: true + storage: false + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/pkg/client/applyconfigurations.go b/pkg/client/applyconfigurations.go new file mode 100644 index 0000000000..97192050f9 --- /dev/null +++ b/pkg/client/applyconfigurations.go @@ -0,0 +1,75 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" +) + +type unstructuredApplyConfiguration struct { + *unstructured.Unstructured +} + +func (u *unstructuredApplyConfiguration) IsApplyConfiguration() {} + +// ApplyConfigurationFromUnstructured creates a runtime.ApplyConfiguration from an *unstructured.Unstructured object. +// +// Do not use Unstructured objects here that were generated from API objects, as its impossible to tell +// if a zero value was explicitly set. +func ApplyConfigurationFromUnstructured(u *unstructured.Unstructured) runtime.ApplyConfiguration { + return &unstructuredApplyConfiguration{Unstructured: u} +} + +type applyconfigurationRuntimeObject struct { + runtime.ApplyConfiguration +} + +func (a *applyconfigurationRuntimeObject) GetObjectKind() schema.ObjectKind { + return a +} + +func (a *applyconfigurationRuntimeObject) GroupVersionKind() schema.GroupVersionKind { + return schema.GroupVersionKind{} +} + +func (a *applyconfigurationRuntimeObject) SetGroupVersionKind(gvk schema.GroupVersionKind) {} + +func (a *applyconfigurationRuntimeObject) DeepCopyObject() runtime.Object { + panic("applyconfigurationRuntimeObject does not support DeepCopyObject") +} + +func runtimeObjectFromApplyConfiguration(ac runtime.ApplyConfiguration) runtime.Object { + return &applyconfigurationRuntimeObject{ApplyConfiguration: ac} +} + +func gvkFromApplyConfiguration(ac applyConfiguration) (schema.GroupVersionKind, error) { + var gvk schema.GroupVersionKind + gv, err := schema.ParseGroupVersion(ptr.Deref(ac.GetAPIVersion(), "")) + if err != nil { + return gvk, fmt.Errorf("failed to parse %q as GroupVersion: %w", ptr.Deref(ac.GetAPIVersion(), ""), err) + } + gvk.Group = gv.Group + gvk.Version = gv.Version + gvk.Kind = ptr.Deref(ac.GetKind(), "") + + return gvk, nil +} diff --git a/pkg/client/client.go b/pkg/client/client.go index bbe36c4673..7e38142273 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -18,12 +18,13 @@ package client import ( "context" + "errors" "fmt" + "net/http" "strings" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -35,45 +36,70 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -// WarningHandlerOptions are options for configuring a -// warning handler for the client which is responsible -// for surfacing API Server warnings. -type WarningHandlerOptions struct { - // SuppressWarnings decides if the warnings from the - // API server are suppressed or surfaced in the client. - SuppressWarnings bool - // AllowDuplicateLogs does not deduplicate the to-be - // logged surfaced warnings messages. See - // log.WarningHandlerOptions for considerations - // regarding deuplication - AllowDuplicateLogs bool -} - // Options are creation options for a Client. type Options struct { + // HTTPClient is the HTTP client to use for requests. + HTTPClient *http.Client + // Scheme, if provided, will be used to map go structs to GroupVersionKinds Scheme *runtime.Scheme // Mapper, if provided, will be used to map GroupVersionKinds to Resources Mapper meta.RESTMapper - // Opts is used to configure the warning handler responsible for - // surfacing and handling warnings messages sent by the API server. - Opts WarningHandlerOptions + // Cache, if provided, is used to read objects from the cache. + Cache *CacheOptions + + // DryRun instructs the client to only perform dry run requests. + DryRun *bool } +// CacheOptions are options for creating a cache-backed client. +type CacheOptions struct { + // Reader is a cache-backed reader that will be used to read objects from the cache. + // +required + Reader Reader + // DisableFor is a list of objects that should never be read from the cache. + // Objects configured here always result in a live lookup. + DisableFor []Object + // Unstructured is a flag that indicates whether the cache-backed client should + // read unstructured objects or lists from the cache. + // If false, unstructured objects will always result in a live lookup. + Unstructured bool +} + +// NewClientFunc allows a user to define how to create a client. +type NewClientFunc func(config *rest.Config, options Options) (Client, error) + // New returns a new Client using the provided config and Options. -// The returned client reads *and* writes directly from the server -// (it doesn't use object caches). It understands how to work with -// normal types (both custom resources and aggregated/built-in resources), -// as well as unstructured types. // +// By default, the client surfaces warnings returned by the server. To +// suppress warnings, set config.WarningHandlerWithContext = rest.NoWarnings{}. To +// define custom behavior, implement the rest.WarningHandlerWithContext interface. +// See [sigs.k8s.io/controller-runtime/pkg/log.KubeAPIWarningLogger] for +// an example. +// +// The client's read behavior is determined by Options.Cache. +// If either Options.Cache or Options.Cache.Reader is nil, +// the client reads directly from the API server. +// If both Options.Cache and Options.Cache.Reader are non-nil, +// the client reads from a local cache. However, specific +// resources can still be configured to bypass the cache based +// on Options.Cache.Unstructured and Options.Cache.DisableFor. +// Write operations are always performed directly on the API server. +// +// The client understands how to work with normal types (both custom resources +// and aggregated/built-in resources), as well as unstructured types. // In the case of normal types, the scheme will be used to look up the // corresponding group, version, and kind for the given type. In the // case of unstructured types, the group, version, and kind will be extracted // from the corresponding fields on the object. -func New(config *rest.Config, options Options) (Client, error) { - return newClient(config, options) +func New(config *rest.Config, options Options) (c Client, err error) { + c, err = newClient(config, options) + if err == nil && options.DryRun != nil && *options.DryRun { + c = NewDryRunClient(c) + } + return c, err } func newClient(config *rest.Config, options Options) (*client, error) { @@ -81,23 +107,29 @@ func newClient(config *rest.Config, options Options) (*client, error) { return nil, fmt.Errorf("must provide non-nil rest.Config to client.New") } - if !options.Opts.SuppressWarnings { - // surface warnings - logger := log.Log.WithName("KubeAPIWarningLogger") - // Set a WarningHandler, the default WarningHandler - // is log.KubeAPIWarningLogger with deduplication enabled. - // See log.KubeAPIWarningLoggerOptions for considerations - // regarding deduplication. - rest.SetDefaultWarningHandler( - log.NewKubeAPIWarningLogger( - logger, - log.KubeAPIWarningLoggerOptions{ - Deduplicate: !options.Opts.AllowDuplicateLogs, - }, - ), + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + if config.WarningHandler == nil && config.WarningHandlerWithContext == nil { + // By default, we surface warnings. + config.WarningHandlerWithContext = log.NewKubeAPIWarningLogger( + log.KubeAPIWarningLoggerOptions{ + Deduplicate: false, + }, ) } + // Use the rest HTTP client for the provided config if unset + if options.HTTPClient == nil { + var err error + options.HTTPClient, err = rest.HTTPClientFor(config) + if err != nil { + return nil, err + } + } + // Init a scheme if none provided if options.Scheme == nil { options.Scheme = scheme.Scheme @@ -106,34 +138,35 @@ func newClient(config *rest.Config, options Options) (*client, error) { // Init a Mapper if none provided if options.Mapper == nil { var err error - options.Mapper, err = apiutil.NewDynamicRESTMapper(config) + options.Mapper, err = apiutil.NewDynamicRESTMapper(config, options.HTTPClient) if err != nil { return nil, err } } - clientcache := &clientCache{ - config: config, - scheme: options.Scheme, - mapper: options.Mapper, - codecs: serializer.NewCodecFactory(options.Scheme), + resources := &clientRestResources{ + httpClient: options.HTTPClient, + config: config, + scheme: options.Scheme, + mapper: options.Mapper, + codecs: serializer.NewCodecFactory(options.Scheme), structuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta), unstructuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta), } - rawMetaClient, err := metadata.NewForConfig(config) + rawMetaClient, err := metadata.NewForConfigAndClient(metadata.ConfigFor(config), options.HTTPClient) if err != nil { return nil, fmt.Errorf("unable to construct metadata-only client for use as part of client: %w", err) } c := &client{ typedClient: typedClient{ - cache: clientcache, + resources: resources, paramCodec: runtime.NewParameterCodec(options.Scheme), }, unstructuredClient: unstructuredClient{ - cache: clientcache, + resources: resources, paramCodec: noConversionParamCodec{}, }, metadataClient: metadataClient{ @@ -143,20 +176,66 @@ func newClient(config *rest.Config, options Options) (*client, error) { scheme: options.Scheme, mapper: options.Mapper, } + if options.Cache == nil || options.Cache.Reader == nil { + return c, nil + } + + // We want a cache if we're here. + // Set the cache. + c.cache = options.Cache.Reader + // Load uncached GVKs. + c.cacheUnstructured = options.Cache.Unstructured + c.uncachedGVKs = map[schema.GroupVersionKind]struct{}{} + for _, obj := range options.Cache.DisableFor { + gvk, err := c.GroupVersionKindFor(obj) + if err != nil { + return nil, err + } + c.uncachedGVKs[gvk] = struct{}{} + } return c, nil } var _ Client = &client{} -// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes -// new clients at the time they are used, and caches the client. +// client is a client.Client configured to either read from a local cache or directly from the API server. +// Write operations are always performed directly on the API server. +// It lazily initializes new clients at the time they are used. type client struct { typedClient typedClient unstructuredClient unstructuredClient metadataClient metadataClient scheme *runtime.Scheme mapper meta.RESTMapper + + cache Reader + uncachedGVKs map[schema.GroupVersionKind]struct{} + cacheUnstructured bool +} + +func (c *client) shouldBypassCache(obj runtime.Object) (bool, error) { + if c.cache == nil { + return true, nil + } + + gvk, err := c.GroupVersionKindFor(obj) + if err != nil { + return false, err + } + // TODO: this is producing unsafe guesses that don't actually work, + // but it matches ~99% of the cases out there. + if meta.IsListType(obj) { + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + } + if _, isUncached := c.uncachedGVKs[gvk]; isUncached { + return true, nil + } + if !c.cacheUnstructured { + _, isUnstructured := obj.(runtime.Unstructured) + return isUnstructured, nil + } + return false, nil } // resetGroupVersionKind is a helper function to restore and preserve GroupVersionKind on an object. @@ -168,6 +247,16 @@ func (c *client) resetGroupVersionKind(obj runtime.Object, gvk schema.GroupVersi } } +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (c *client) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return apiutil.GVKForObject(obj, c.scheme) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (c *client) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return apiutil.IsObjectNamespaced(obj, c.scheme, c.mapper) +} + // Scheme returns the scheme this client is using. func (c *client) Scheme() *runtime.Scheme { return c.scheme @@ -181,7 +270,7 @@ func (c *client) RESTMapper() meta.RESTMapper { // Create implements client.Client. func (c *client) Create(ctx context.Context, obj Object, opts ...CreateOption) error { switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Create(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return fmt.Errorf("cannot create using only metadata") @@ -194,7 +283,7 @@ func (c *client) Create(ctx context.Context, obj Object, opts ...CreateOption) e func (c *client) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Update(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return fmt.Errorf("cannot update using only metadata -- did you mean to patch?") @@ -206,7 +295,7 @@ func (c *client) Update(ctx context.Context, obj Object, opts ...UpdateOption) e // Delete implements client.Client. func (c *client) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Delete(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return c.metadataClient.Delete(ctx, obj, opts...) @@ -218,7 +307,7 @@ func (c *client) Delete(ctx context.Context, obj Object, opts ...DeleteOption) e // DeleteAllOf implements client.Client. func (c *client) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.DeleteAllOf(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return c.metadataClient.DeleteAllOf(ctx, obj, opts...) @@ -231,7 +320,7 @@ func (c *client) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllO func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Patch(ctx, obj, patch, opts...) case *metav1.PartialObjectMetadata: return c.metadataClient.Patch(ctx, obj, patch, opts...) @@ -240,24 +329,50 @@ func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...Pat } } +func (c *client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + switch obj := obj.(type) { + case *unstructuredApplyConfiguration: + defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + return c.unstructuredClient.Apply(ctx, obj, opts...) + default: + return c.typedClient.Apply(ctx, obj, opts...) + } +} + // Get implements client.Client. -func (c *client) Get(ctx context.Context, key ObjectKey, obj Object) error { +func (c *client) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + if isUncached, err := c.shouldBypassCache(obj); err != nil { + return err + } else if !isUncached { + // Attempt to get from the cache. + return c.cache.Get(ctx, key, obj, opts...) + } + + // Perform a live lookup. switch obj.(type) { - case *unstructured.Unstructured: - return c.unstructuredClient.Get(ctx, key, obj) + case runtime.Unstructured: + return c.unstructuredClient.Get(ctx, key, obj, opts...) case *metav1.PartialObjectMetadata: // Metadata only object should always preserve the GVK coming in from the caller. defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) - return c.metadataClient.Get(ctx, key, obj) + return c.metadataClient.Get(ctx, key, obj, opts...) default: - return c.typedClient.Get(ctx, key, obj) + return c.typedClient.Get(ctx, key, obj, opts...) } } // List implements client.Client. func (c *client) List(ctx context.Context, obj ObjectList, opts ...ListOption) error { + if isUncached, err := c.shouldBypassCache(obj); err != nil { + return err + } else if !isUncached { + // Attempt to get from the cache. + return c.cache.List(ctx, obj, opts...) + } + + // Perform a live lookup. switch x := obj.(type) { - case *unstructured.UnstructuredList: + case runtime.Unstructured: return c.unstructuredClient.List(ctx, obj, opts...) case *metav1.PartialObjectMetadataList: // Metadata only object should always preserve the GVK. @@ -289,40 +404,228 @@ func (c *client) List(ctx context.Context, obj ObjectList, opts ...ListOption) e } // Status implements client.StatusClient. -func (c *client) Status() StatusWriter { - return &statusWriter{client: c} +func (c *client) Status() SubResourceWriter { + return c.SubResource("status") +} + +func (c *client) SubResource(subResource string) SubResourceClient { + return &subResourceClient{client: c, subResource: subResource} +} + +// subResourceClient is client.SubResourceWriter that writes to subresources. +type subResourceClient struct { + client *client + subResource string +} + +// ensure subResourceClient implements client.SubResourceClient. +var _ SubResourceClient = &subResourceClient{} + +// SubResourceGetOptions holds all the possible configuration +// for a subresource Get request. +type SubResourceGetOptions struct { + Raw *metav1.GetOptions } -// statusWriter is client.StatusWriter that writes status subresource. -type statusWriter struct { - client *client +// ApplyToSubResourceGet updates the configuaration to the given get options. +func (getOpt *SubResourceGetOptions) ApplyToSubResourceGet(o *SubResourceGetOptions) { + if getOpt.Raw != nil { + o.Raw = getOpt.Raw + } +} + +// ApplyOptions applues the given options. +func (getOpt *SubResourceGetOptions) ApplyOptions(opts []SubResourceGetOption) *SubResourceGetOptions { + for _, o := range opts { + o.ApplyToSubResourceGet(getOpt) + } + + return getOpt } -// ensure statusWriter implements client.StatusWriter. -var _ StatusWriter = &statusWriter{} +// AsGetOptions returns the configured options as *metav1.GetOptions. +func (getOpt *SubResourceGetOptions) AsGetOptions() *metav1.GetOptions { + if getOpt.Raw == nil { + return &metav1.GetOptions{} + } + return getOpt.Raw +} -// Update implements client.StatusWriter. -func (sw *statusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) +// SubResourceUpdateOptions holds all the possible configuration +// for a subresource update request. +type SubResourceUpdateOptions struct { + UpdateOptions + SubResourceBody Object +} + +// ApplyToSubResourceUpdate updates the configuration on the given create options +func (uo *SubResourceUpdateOptions) ApplyToSubResourceUpdate(o *SubResourceUpdateOptions) { + uo.UpdateOptions.ApplyToUpdate(&o.UpdateOptions) + if uo.SubResourceBody != nil { + o.SubResourceBody = uo.SubResourceBody + } +} + +// ApplyOptions applies the given options. +func (uo *SubResourceUpdateOptions) ApplyOptions(opts []SubResourceUpdateOption) *SubResourceUpdateOptions { + for _, o := range opts { + o.ApplyToSubResourceUpdate(uo) + } + + return uo +} + +// SubResourceUpdateAndPatchOption is an option that can be used for either +// a subresource update or patch request. +type SubResourceUpdateAndPatchOption interface { + SubResourceUpdateOption + SubResourcePatchOption +} + +// WithSubResourceBody returns an option that uses the given body +// for a subresource Update or Patch operation. +func WithSubResourceBody(body Object) SubResourceUpdateAndPatchOption { + return &withSubresourceBody{body: body} +} + +type withSubresourceBody struct { + body Object +} + +func (wsr *withSubresourceBody) ApplyToSubResourceUpdate(o *SubResourceUpdateOptions) { + o.SubResourceBody = wsr.body +} + +func (wsr *withSubresourceBody) ApplyToSubResourcePatch(o *SubResourcePatchOptions) { + o.SubResourceBody = wsr.body +} + +// SubResourceCreateOptions are all the possible configurations for a subresource +// create request. +type SubResourceCreateOptions struct { + CreateOptions +} + +// ApplyOptions applies the given options. +func (co *SubResourceCreateOptions) ApplyOptions(opts []SubResourceCreateOption) *SubResourceCreateOptions { + for _, o := range opts { + o.ApplyToSubResourceCreate(co) + } + + return co +} + +// ApplyToSubResourceCreate applies the the configuration on the given create options. +func (co *SubResourceCreateOptions) ApplyToSubResourceCreate(o *SubResourceCreateOptions) { + co.CreateOptions.ApplyToCreate(&co.CreateOptions) +} + +// SubResourcePatchOptions holds all possible configurations for a subresource patch +// request. +type SubResourcePatchOptions struct { + PatchOptions + SubResourceBody Object +} + +// ApplyOptions applies the given options. +func (po *SubResourcePatchOptions) ApplyOptions(opts []SubResourcePatchOption) *SubResourcePatchOptions { + for _, o := range opts { + o.ApplyToSubResourcePatch(po) + } + + return po +} + +// ApplyToSubResourcePatch applies the configuration on the given patch options. +func (po *SubResourcePatchOptions) ApplyToSubResourcePatch(o *SubResourcePatchOptions) { + po.PatchOptions.ApplyToPatch(&o.PatchOptions) + if po.SubResourceBody != nil { + o.SubResourceBody = po.SubResourceBody + } +} + +// SubResourceApplyOptions are the options for a subresource +// apply request. +type SubResourceApplyOptions struct { + ApplyOptions + SubResourceBody runtime.ApplyConfiguration +} + +// ApplyOpts applies the given options. +func (ao *SubResourceApplyOptions) ApplyOpts(opts []SubResourceApplyOption) *SubResourceApplyOptions { + for _, o := range opts { + o.ApplyToSubResourceApply(ao) + } + + return ao +} + +// ApplyToSubResourceApply applies the configuration on the given patch options. +func (ao *SubResourceApplyOptions) ApplyToSubResourceApply(o *SubResourceApplyOptions) { + ao.ApplyOptions.ApplyToApply(&o.ApplyOptions) + if ao.SubResourceBody != nil { + o.SubResourceBody = ao.SubResourceBody + } +} + +func (sc *subResourceClient) Get(ctx context.Context, obj Object, subResource Object, opts ...SubResourceGetOption) error { switch obj.(type) { - case *unstructured.Unstructured: - return sw.client.unstructuredClient.UpdateStatus(ctx, obj, opts...) + case runtime.Unstructured: + return sc.client.unstructuredClient.GetSubResource(ctx, obj, subResource, sc.subResource, opts...) + case *metav1.PartialObjectMetadata: + return errors.New("can not get subresource using only metadata") + default: + return sc.client.typedClient.GetSubResource(ctx, obj, subResource, sc.subResource, opts...) + } +} + +// Create implements client.SubResourceClient +func (sc *subResourceClient) Create(ctx context.Context, obj Object, subResource Object, opts ...SubResourceCreateOption) error { + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + defer sc.client.resetGroupVersionKind(subResource, subResource.GetObjectKind().GroupVersionKind()) + + switch obj.(type) { + case runtime.Unstructured: + return sc.client.unstructuredClient.CreateSubResource(ctx, obj, subResource, sc.subResource, opts...) case *metav1.PartialObjectMetadata: return fmt.Errorf("cannot update status using only metadata -- did you mean to patch?") default: - return sw.client.typedClient.UpdateStatus(ctx, obj, opts...) + return sc.client.typedClient.CreateSubResource(ctx, obj, subResource, sc.subResource, opts...) } } -// Patch implements client.Client. -func (sw *statusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) +// Update implements client.SubResourceClient +func (sc *subResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) switch obj.(type) { - case *unstructured.Unstructured: - return sw.client.unstructuredClient.PatchStatus(ctx, obj, patch, opts...) + case runtime.Unstructured: + return sc.client.unstructuredClient.UpdateSubResource(ctx, obj, sc.subResource, opts...) case *metav1.PartialObjectMetadata: - return sw.client.metadataClient.PatchStatus(ctx, obj, patch, opts...) + return fmt.Errorf("cannot update status using only metadata -- did you mean to patch?") + default: + return sc.client.typedClient.UpdateSubResource(ctx, obj, sc.subResource, opts...) + } +} + +// Patch implements client.SubResourceWriter. +func (sc *subResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + switch obj.(type) { + case runtime.Unstructured: + return sc.client.unstructuredClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...) + case *metav1.PartialObjectMetadata: + return sc.client.metadataClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...) + default: + return sc.client.typedClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...) + } +} + +func (sc *subResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + switch obj := obj.(type) { + case *unstructuredApplyConfiguration: + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + return sc.client.unstructuredClient.ApplySubResource(ctx, obj, sc.subResource, opts...) default: - return sw.client.typedClient.PatchStatus(ctx, obj, patch, opts...) + return sc.client.typedClient.ApplySubResource(ctx, obj, sc.subResource, opts...) } } diff --git a/pkg/client/client_cache.go b/pkg/client/client_rest_resources.go similarity index 58% rename from pkg/client/client_cache.go rename to pkg/client/client_rest_resources.go index 857a0b38a7..acff7a46a4 100644 --- a/pkg/client/client_cache.go +++ b/pkg/client/client_rest_resources.go @@ -17,21 +17,25 @@ limitations under the License. package client import ( + "fmt" + "net/http" "strings" "sync" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/rest" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) -// clientCache creates and caches rest clients and metadata for Kubernetes types. -type clientCache struct { +// clientRestResources creates and stores rest clients and metadata for Kubernetes types. +type clientRestResources struct { + // httpClient is the http client to use for requests + httpClient *http.Client + // config is the rest.Config to talk to an apiserver config *rest.Config @@ -44,22 +48,26 @@ type clientCache struct { // codecs are used to create a REST client for a gvk codecs serializer.CodecFactory - // structuredResourceByType caches structured type metadata + // structuredResourceByType stores structured type metadata structuredResourceByType map[schema.GroupVersionKind]*resourceMeta - // unstructuredResourceByType caches unstructured type metadata + // unstructuredResourceByType stores unstructured type metadata unstructuredResourceByType map[schema.GroupVersionKind]*resourceMeta mu sync.RWMutex } // newResource maps obj to a Kubernetes Resource and constructs a client for that Resource. // If the object is a list, the resource represents the item's type instead. -func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) { +func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, + isList bool, + forceDisableProtoBuf bool, + isUnstructured bool, +) (*resourceMeta, error) { if strings.HasSuffix(gvk.Kind, "List") && isList { // if this was a list, treat it as a request for the item's resource gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] } - client, err := apiutil.RESTClientForGVK(gvk, isUnstructured, c.config, c.codecs) + client, err := apiutil.RESTClientForGVK(gvk, forceDisableProtoBuf, isUnstructured, c.config, c.codecs, c.httpClient) if err != nil { return nil, err } @@ -70,17 +78,41 @@ func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList, isUnstruc return &resourceMeta{Interface: client, mapping: mapping, gvk: gvk}, nil } +type applyConfiguration interface { + GetName() *string + GetNamespace() *string + GetKind() *string + GetAPIVersion() *string +} + // getResource returns the resource meta information for the given type of object. // If the object is a list, the resource represents the item's type instead. -func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) { - gvk, err := apiutil.GVKForObject(obj, c.scheme) - if err != nil { - return nil, err +func (c *clientRestResources) getResource(obj any) (*resourceMeta, error) { + var gvk schema.GroupVersionKind + var err error + var isApplyConfiguration bool + switch o := obj.(type) { + case runtime.Object: + gvk, err = apiutil.GVKForObject(o, c.scheme) + if err != nil { + return nil, err + } + case runtime.ApplyConfiguration: + ac, ok := o.(applyConfiguration) + if !ok { + return nil, fmt.Errorf("%T is a runtime.ApplyConfiguration but not an applyConfiguration", o) + } + gvk, err = gvkFromApplyConfiguration(ac) + if err != nil { + return nil, err + } + isApplyConfiguration = true + default: + return nil, fmt.Errorf("bug: %T is neither a runtime.Object nor a runtime.ApplyConfiguration", o) } - _, isUnstructured := obj.(*unstructured.Unstructured) - _, isUnstructuredList := obj.(*unstructured.UnstructuredList) - isUnstructured = isUnstructured || isUnstructuredList + _, isUnstructured := obj.(runtime.Unstructured) + forceDisableProtoBuf := isUnstructured || isApplyConfiguration // It's better to do creation work twice than to not let multiple // people make requests at once @@ -96,10 +128,15 @@ func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) { return r, nil } + var isList bool + if runtimeObject, ok := obj.(runtime.Object); ok && meta.IsListType(runtimeObject) { + isList = true + } + // Initialize a new Client c.mu.Lock() defer c.mu.Unlock() - r, err = c.newResource(gvk, meta.IsListType(obj), isUnstructured) + r, err = c.newResource(gvk, isList, forceDisableProtoBuf, isUnstructured) if err != nil { return nil, err } @@ -108,19 +145,32 @@ func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) { } // getObjMeta returns objMeta containing both type and object metadata and state. -func (c *clientCache) getObjMeta(obj runtime.Object) (*objMeta, error) { +func (c *clientRestResources) getObjMeta(obj any) (*objMeta, error) { r, err := c.getResource(obj) if err != nil { return nil, err } - m, err := meta.Accessor(obj) - if err != nil { - return nil, err + objMeta := &objMeta{resourceMeta: r} + + switch o := obj.(type) { + case runtime.Object: + m, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + objMeta.namespace = m.GetNamespace() + objMeta.name = m.GetName() + case applyConfiguration: + objMeta.namespace = ptr.Deref(o.GetNamespace(), "") + objMeta.name = ptr.Deref(o.GetName(), "") + default: + return nil, fmt.Errorf("object %T is neither a runtime.Object nor a runtime.ApplyConfiguration", obj) } - return &objMeta{resourceMeta: r, Object: m}, err + + return objMeta, nil } -// resourceMeta caches state for a Kubernetes type. +// resourceMeta stores state for a Kubernetes type. type resourceMeta struct { // client is the rest client used to talk to the apiserver rest.Interface @@ -145,6 +195,6 @@ type objMeta struct { // resourceMeta contains type information for the object *resourceMeta - // Object contains meta data for the object instance - metav1.Object + namespace string + name string } diff --git a/pkg/client/client_suite_test.go b/pkg/client/client_suite_test.go index c7ed32e7bc..89cab3f7ed 100644 --- a/pkg/client/client_suite_test.go +++ b/pkg/client/client_suite_test.go @@ -17,33 +17,46 @@ limitations under the License. package client_test import ( + "bytes" + "io" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/examples/crd/pkg" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) -func TestSource(t *testing.T) { +func TestClient(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Client Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Client Suite") } -var testenv *envtest.Environment -var cfg *rest.Config -var clientset *kubernetes.Clientset +var ( + testenv *envtest.Environment + cfg *rest.Config + clientset *kubernetes.Clientset + + // Used by tests to inspect controller and client log messages. + log bytes.Buffer +) var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + // Forwards logs to ginkgo output, and allows tests to inspect logs. + mw := io.MultiWriter(&log, GinkgoWriter) + + // Use prefixes to help us tell the source of the log message. + // controller-runtime uses logf + logf.SetLogger(zap.New(zap.WriteTo(mw), zap.UseDevMode(true)).WithName("logf")) + // client-go logs uses klog + klog.SetLogger(zap.New(zap.WriteTo(mw), zap.UseDevMode(true)).WithName("klog")) testenv = &envtest.Environment{CRDDirectoryPaths: []string{"./testdata"}} @@ -55,7 +68,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(pkg.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred()) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index fac8c97ab8..42e14771cc 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -17,29 +17,44 @@ limitations under the License. package client_test import ( + "bufio" + "bytes" "context" + "encoding/json" + "errors" "fmt" + "reflect" + "strings" "sync/atomic" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + appsv1applyconfigurations "k8s.io/client-go/applyconfigurations/apps/v1" + autoscaling1applyconfigurations "k8s.io/client-go/applyconfigurations/autoscaling/v1" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" kscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/examples/crd/pkg" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" ) -const serverSideTimeoutSeconds = 10 - func deleteDeployment(ctx context.Context, dep *appsv1.Deployment, ns string) { _, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) if err == nil { @@ -132,10 +147,12 @@ var _ = Describe("Client", func() { var dep *appsv1.Deployment var pod *corev1.Pod var node *corev1.Node + var serviceAccount *corev1.ServiceAccount + var csr *certificatesv1.CertificateSigningRequest var count uint64 = 0 var replicaCount int32 = 2 var ns = "default" - ctx := context.TODO() + var errNotCached *cache.ErrResourceNotCached BeforeEach(func() { atomic.AddUint64(&count, 1) @@ -166,11 +183,35 @@ var _ = Describe("Client", func() { ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("node-name-%v", count)}, Spec: corev1.NodeSpec{}, } + serviceAccount = &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("sa-%v", count), Namespace: ns}} + csr = &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("csr-%v", count)}, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "org.io/my-signer", + Request: []byte(`-----BEGIN CERTIFICATE REQUEST----- +MIIChzCCAW8CAQAwQjELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0 +eTEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBANe06dLX/bDNm6mVEnKdJexcJM6WKMFSt5o6BEdD1+Ki +WyUcvfNgIBbwAZjkF9U1r7+KuDcc6XYFnb6ky1wPo4C+XwcIIx7Nnbf8IdWJukPb +2BCsqO4NCsG6kKFavmH9J3q//nwKUvlQE+AJ2MPuOAZTwZ4KskghiGuS8hyk6/PZ +XH9QhV7Jma43bDzQozd2C7OujRBhLsuP94KSu839RRFWd9ms3XHgTxLxb7nxwZDx +9l7/ZVAObJoQYlHENqs12NCVP4gpJfbcY8/rd+IG4ftcZEmpeO4kKO+d2TpRKQqw +bjCMoAdD5Y43iLTtyql4qRnbMe3nxYG2+1inEryuV/cCAwEAAaAAMA0GCSqGSIb3 +DQEBCwUAA4IBAQDH5hDByRN7wERQtC/o6uc8Y+yhjq9YcBJjjbnD6Vwru5pOdWtx +qfKkkXI5KNOdEhWzLnJyOcWHjj8UoHqI3AjxGC7dTM95eGjxQGUpsUOX8JSd4MiZ +cct4g4BKBj02AGqZLiEgN+PLCYAmEaYU7oZc4OAh6WzMrljNRsj66awMQpw8O1eY +YuBa8vwz8ko8vn/pn7IrFu8cZ+EA3rluJ+budX/QrEGi1hijg27q7/Qr0wNI9f1v +086mLKdqaBTkblXWEvF3WP4CcLNyrSNi4eu+G0fcAgGp1F/Nqh0MuWKSOLprv5Om +U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC +-----END CERTIFICATE REQUEST-----`), + Usages: []certificatesv1.KeyUsage{certificatesv1.UsageClientAuth}, + }, + } scheme = kscheme.Scheme - }, serverSideTimeoutSeconds) + }) var delOptions *metav1.DeleteOptions - AfterEach(func() { + AfterEach(func(ctx SpecContext) { // Cleanup var zero int64 = 0 policy := metav1.DeletePropagationForeground @@ -184,10 +225,72 @@ var _ = Describe("Client", func() { err = clientset.CoreV1().Nodes().Delete(ctx, node.Name, *delOptions) Expect(err).NotTo(HaveOccurred()) } - }, serverSideTimeoutSeconds) + err = clientset.CoreV1().ServiceAccounts(ns).Delete(ctx, serviceAccount.Name, *delOptions) + Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred()) + + err = clientset.CertificatesV1().CertificateSigningRequests().Delete(ctx, csr.Name, *delOptions) + Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred()) + }) + + Describe("WarningHandler", func() { + It("should log warnings with config.WarningHandler, if one is defined", func(ctx SpecContext) { + cache := &fakeReader{} + + testCfg := rest.CopyConfig(cfg) + + var testLog bytes.Buffer + testCfg.WarningHandler = rest.NewWarningWriter(&testLog, rest.WarningWriterOptions{}) + + cl, err := client.New(testCfg, client.Options{Cache: &client.CacheOptions{Reader: cache, DisableFor: []client.Object{&corev1.Namespace{}}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "wh-defined"}} + tns, err = clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(tns).NotTo(BeNil()) + defer deleteNamespace(ctx, tns) + + toCreate := &pkg.ChaosPod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: tns.Name, + }, + // The ChaosPod CRD does not define Status, so the field is unknown to the API server, + // but field validation is not strict by default, so the API server returns a warning, + // and we need a warning to check whether suppression works. + Status: pkg.ChaosPodStatus{}, + } + err = cl.Create(ctx, toCreate) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + scannerTestLog := bufio.NewScanner(&testLog) + for scannerTestLog.Scan() { + line := scannerTestLog.Text() + if strings.Contains( + line, + "unknown field \"status\"", + ) { + return + } + } + defer Fail("expected to find one API server warning logged the config.WarningHandler") + + scanner := bufio.NewScanner(&log) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains( + line, + "unknown field \"status\"", + ) { + defer Fail("expected to find zero API server warnings in the client log") + break + } + } + }) + }) - // TODO(seans): Cast "cl" as "client" struct from "Client" interface. Then validate the - // instance values for the "client" struct. Describe("New", func() { It("should return a new Client", func() { cl, err := client.New(cfg, client.Options{}) @@ -201,41 +304,78 @@ var _ = Describe("Client", func() { Expect(cl).To(BeNil()) }) - // TODO(seans): cast as client struct and inspect Scheme It("should use the provided Scheme if provided", func() { cl, err := client.New(cfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) + Expect(cl.Scheme()).ToNot(BeNil()) + Expect(cl.Scheme()).To(Equal(scheme)) }) - // TODO(seans): cast as client struct and inspect Scheme It("should default the Scheme if not provided", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) + Expect(cl.Scheme()).ToNot(BeNil()) + Expect(cl.Scheme()).To(Equal(kscheme.Scheme)) }) - PIt("should use the provided Mapper if provided", func() { - + It("should use the provided Mapper if provided", func() { + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + cl, err := client.New(cfg, client.Options{Mapper: mapper}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + Expect(cl.RESTMapper()).ToNot(BeNil()) + Expect(cl.RESTMapper()).To(Equal(mapper)) }) - // TODO(seans): cast as client struct and inspect Mapper It("should create a Mapper if not provided", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) + Expect(cl.RESTMapper()).ToNot(BeNil()) + }) + + It("should use the provided reader cache if provided, on get and list", func(ctx SpecContext) { + cache := &fakeReader{} + cl, err := client.New(cfg, client.Options{Cache: &client.CacheOptions{Reader: cache}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + Expect(cl.Get(ctx, client.ObjectKey{Name: "test"}, &appsv1.Deployment{})).To(Succeed()) + Expect(cl.List(ctx, &appsv1.DeploymentList{})).To(Succeed()) + Expect(cache.Called).To(Equal(2)) + }) + + It("should propagate ErrResourceNotCached errors", func(ctx SpecContext) { + c := &fakeUncachedReader{} + cl, err := client.New(cfg, client.Options{Cache: &client.CacheOptions{Reader: c}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + Expect(errors.As(cl.Get(ctx, client.ObjectKey{Name: "test"}, &appsv1.Deployment{}), &errNotCached)).To(BeTrue()) + Expect(errors.As(cl.List(ctx, &appsv1.DeploymentList{}), &errNotCached)).To(BeTrue()) + Expect(c.Called).To(Equal(2)) + }) + + It("should not use the provided reader cache if provided, on get and list for uncached GVKs", func(ctx SpecContext) { + cache := &fakeReader{} + cl, err := client.New(cfg, client.Options{Cache: &client.CacheOptions{Reader: cache, DisableFor: []client.Object{&corev1.Namespace{}}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + Expect(cl.Get(ctx, client.ObjectKey{Name: "default"}, &corev1.Namespace{})).To(Succeed()) + Expect(cl.List(ctx, &corev1.NamespaceList{})).To(Succeed()) + Expect(cache.Called).To(Equal(0)) }) }) Describe("Create", func() { Context("with structured objects", func() { - It("should create a new object from a go struct", func() { + It("should create a new object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the object") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -246,13 +386,13 @@ var _ = Describe("Client", func() { Expect(dep).To(Equal(actual)) }) - It("should create a new object non-namespace object from a go struct", func() { + It("should create a new object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the object") - err = cl.Create(context.TODO(), node) + err = cl.Create(ctx, node) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) @@ -263,7 +403,7 @@ var _ = Describe("Client", func() { Expect(node).To(Equal(actual)) }) - It("should fail if the object already exists", func() { + It("should fail if the object already exists", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -271,7 +411,7 @@ var _ = Describe("Client", func() { old := dep.DeepCopy() By("creating the object") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -279,24 +419,24 @@ var _ = Describe("Client", func() { Expect(actual).NotTo(BeNil()) By("creating the object a second time") - err = cl.Create(context.TODO(), old) + err = cl.Create(ctx, old) Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) }) - It("should fail if the object does not pass server-side validation", func() { + It("should fail if the object does not pass server-side validation", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the pod, since required field Containers is empty") - err = cl.Create(context.TODO(), pod) + err = cl.Create(ctx, pod) Expect(err).To(HaveOccurred()) // TODO(seans): Add test to validate the returned error. Problems currently with // different returned error locally versus travis. - }, serverSideTimeoutSeconds) + }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -304,7 +444,7 @@ var _ = Describe("Client", func() { Expect(cl).NotTo(BeNil()) By("creating the object fails") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -315,13 +455,28 @@ var _ = Describe("Client", func() { }) Context("with the DryRun option", func() { - It("should not create a new object", func() { + It("should not create a new object, global option", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{DryRun: ptr.To(true)}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("creating the object (with DryRun)") + err = cl.Create(ctx, dep) + Expect(err).NotTo(HaveOccurred()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Expect(actual).To(Equal(&appsv1.Deployment{})) + }) + + It("should not create a new object, inline option", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the object (with DryRun)") - err = cl.Create(context.TODO(), dep, client.DryRunAll) + err = cl.Create(ctx, dep, client.DryRunAll) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -333,7 +488,7 @@ var _ = Describe("Client", func() { }) Context("with unstructured objects", func() { - It("should create a new object from a go struct", func() { + It("should create a new object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -348,7 +503,7 @@ var _ = Describe("Client", func() { }) By("creating the object") - err = cl.Create(context.TODO(), u) + err = cl.Create(ctx, u) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -356,7 +511,7 @@ var _ = Describe("Client", func() { Expect(actual).NotTo(BeNil()) }) - It("should create a new non-namespace object ", func() { + It("should create a new non-namespace object ", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -371,7 +526,7 @@ var _ = Describe("Client", func() { }) By("creating the object") - err = cl.Create(context.TODO(), node) + err = cl.Create(ctx, node) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) @@ -385,7 +540,7 @@ var _ = Describe("Client", func() { Expect(u).To(Equal(au)) }) - It("should fail if the object already exists", func() { + It("should fail if the object already exists", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -393,7 +548,7 @@ var _ = Describe("Client", func() { old := dep.DeepCopy() By("creating the object") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -409,12 +564,12 @@ var _ = Describe("Client", func() { }) By("creating the object a second time") - err = cl.Create(context.TODO(), u) + err = cl.Create(ctx, u) Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) }) - It("should fail if the object does not pass server-side validation", func() { + It("should fail if the object does not pass server-side validation", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -427,26 +582,26 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "Pod", }) - err = cl.Create(context.TODO(), u) + err = cl.Create(ctx, u) Expect(err).To(HaveOccurred()) // TODO(seans): Add test to validate the returned error. Problems currently with // different returned error locally versus travis. - }, serverSideTimeoutSeconds) + }) }) Context("with metadata objects", func() { - It("should fail with an error", func() { + It("should fail with an error", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) obj := metaOnlyFromObj(dep, scheme) - Expect(cl.Create(context.TODO(), obj)).NotTo(Succeed()) + Expect(cl.Create(ctx, obj)).NotTo(Succeed()) }) }) Context("with the DryRun option", func() { - It("should not create a new object from a go struct", func() { + It("should not create a new object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -461,7 +616,7 @@ var _ = Describe("Client", func() { }) By("creating the object") - err = cl.Create(context.TODO(), u, client.DryRunAll) + err = cl.Create(ctx, u, client.DryRunAll) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -474,7 +629,7 @@ var _ = Describe("Client", func() { Describe("Update", func() { Context("with structured objects", func() { - It("should update an existing object from a go struct", func() { + It("should update an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -485,7 +640,7 @@ var _ = Describe("Client", func() { By("updating the Deployment") dep.Annotations = map[string]string{"foo": "bar"} - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new annotation") @@ -495,7 +650,7 @@ var _ = Describe("Client", func() { Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should update and preserve type information", func() { + It("should update and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -506,14 +661,14 @@ var _ = Describe("Client", func() { By("updating the Deployment") dep.SetGroupVersionKind(depGvk) - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) }) - It("should update an existing object non-namespace object from a go struct", func() { + It("should update an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -523,7 +678,7 @@ var _ = Describe("Client", func() { By("updating the object") node.Annotations = map[string]string{"foo": "bar"} - err = cl.Update(context.TODO(), node) + err = cl.Update(ctx, node) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -533,13 +688,13 @@ var _ = Describe("Client", func() { Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("updating non-existent object") - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).To(HaveOccurred()) }) @@ -551,7 +706,7 @@ var _ = Describe("Client", func() { }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -564,7 +719,7 @@ var _ = Describe("Client", func() { By("updating the Deployment") dep.Annotations = map[string]string{"foo": "bar"} - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -574,7 +729,7 @@ var _ = Describe("Client", func() { }) }) Context("with unstructured objects", func() { - It("should update an existing object from a go struct", func() { + It("should update an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -592,7 +747,7 @@ var _ = Describe("Client", func() { Version: "v1", }) u.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.TODO(), u) + err = cl.Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new annotation") @@ -602,7 +757,7 @@ var _ = Describe("Client", func() { Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should update and preserve type information", func() { + It("should update and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -616,14 +771,14 @@ var _ = Describe("Client", func() { Expect(scheme.Convert(dep, u, nil)).To(Succeed()) u.SetGroupVersionKind(depGvk) u.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.TODO(), u) + err = cl.Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(u.GroupVersionKind()).To(Equal(depGvk)) }) - It("should update an existing object non-namespace object from a go struct", func() { + It("should update an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -640,7 +795,7 @@ var _ = Describe("Client", func() { Version: "v1", }) u.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.TODO(), u) + err = cl.Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -649,7 +804,7 @@ var _ = Describe("Client", func() { Expect(actual).NotTo(BeNil()) Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -658,25 +813,25 @@ var _ = Describe("Client", func() { u := &unstructured.Unstructured{} Expect(scheme.Convert(dep, u, nil)).To(Succeed()) u.SetGroupVersionKind(depGvk) - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).To(HaveOccurred()) }) }) Context("with metadata objects", func() { - It("should fail with an error", func() { + It("should fail with an error", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) obj := metaOnlyFromObj(dep, scheme) - Expect(cl.Update(context.TODO(), obj)).NotTo(Succeed()) + Expect(cl.Update(ctx, obj)).NotTo(Succeed()) }) }) }) Describe("Patch", func() { Context("Metadata Client", func() { - It("should merge patch with options", func() { + It("should merge patch with options", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -692,7 +847,7 @@ var _ = Describe("Client", func() { metadata.Labels["foo"] = "bar" testOption := &mockPatchOption{} - Expect(cl.Patch(context.TODO(), metadata, client.Merge, testOption)).To(Succeed()) + Expect(cl.Patch(ctx, metadata, client.Merge, testOption)).To(Succeed()) By("validating that patched metadata has new labels") actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -701,14 +856,596 @@ var _ = Describe("Client", func() { Expect(actual.Labels["foo"]).To(Equal("bar")) By("validating patch options were applied") - Expect(testOption.applied).To(Equal(true)) + Expect(testOption.applied).To(BeTrue()) + }) + }) + }) + + Describe("Apply", func() { + Context("Unstructured Client", func() { + It("should create and update a configMap using SSA", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + data := map[string]any{ + "some-key": "some-value", + } + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": "test-configmap", + "namespace": "default", + }, + "data": data, + }} + + err = cl.Apply(ctx, client.ApplyConfigurationFromUnstructured(obj), &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err := clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Get(ctx, obj.GetName(), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + actualData := map[string]any{} + for k, v := range cm.Data { + actualData[k] = v + } + + Expect(actualData).To(BeComparableTo(data)) + Expect(actualData).To(BeComparableTo(obj.Object["data"])) + + data = map[string]any{ + "a-new-key": "a-new-value", + } + obj.Object["data"] = data + unstructured.RemoveNestedField(obj.Object, "metadata", "managedFields") + + err = cl.Apply(ctx, client.ApplyConfigurationFromUnstructured(obj), &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err = clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Get(ctx, obj.GetName(), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + actualData = map[string]any{} + for k, v := range cm.Data { + actualData[k] = v + } + + Expect(actualData).To(BeComparableTo(data)) + Expect(actualData).To(BeComparableTo(obj.Object["data"])) + }) + }) + + Context("Structured Client", func() { + It("should create and update a configMap using SSA", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + data := map[string]string{ + "some-key": "some-value", + } + obj := corev1applyconfigurations. + ConfigMap("test-configmap", "default"). + WithData(data) + + err = cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err := clientset.CoreV1().ConfigMaps(ptr.Deref(obj.GetNamespace(), "")).Get(ctx, ptr.Deref(obj.GetName(), ""), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(cm.Data).To(BeComparableTo(data)) + Expect(cm.Data).To(BeComparableTo(obj.Data)) + + data = map[string]string{ + "a-new-key": "a-new-value", + } + obj.Data = data + + err = cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err = clientset.CoreV1().ConfigMaps(ptr.Deref(obj.GetNamespace(), "")).Get(ctx, ptr.Deref(obj.GetName(), ""), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(cm.Data).To(BeComparableTo(data)) + Expect(cm.Data).To(BeComparableTo(obj.Data)) + }) + }) + }) + + Describe("SubResourceClient", func() { + Context("with structured objects", func() { + It("should be able to read the Scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("reading the scale subresource") + scale := &autoscalingv1.Scale{} + err = cl.SubResource("scale").Get(ctx, dep, scale, &client.SubResourceGetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.Spec.Replicas).To(Equal(*dep.Spec.Replicas)) + }) + It("should be able to create ServiceAccount tokens", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the serviceAccount") + _, err = clientset.CoreV1().ServiceAccounts(serviceAccount.Namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) + Expect((err)).NotTo(HaveOccurred()) + + token := &authenticationv1.TokenRequest{} + err = cl.SubResource("token").Create(ctx, serviceAccount, token, &client.SubResourceCreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(token.Status.Token).NotTo(Equal("")) + }) + + It("should be able to create Pod evictions", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the eviction") + eviction := &policyv1.Eviction{ + DeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))}, + } + err = cl.SubResource("eviction").Create(ctx, pod, eviction, &client.SubResourceCreateOptions{}) + Expect((err)).NotTo(HaveOccurred()) + + By("Asserting the pod is gone") + _, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should be able to create Pod bindings", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the binding") + binding := &corev1.Binding{ + Target: corev1.ObjectReference{Name: node.Name}, + } + err = cl.SubResource("binding").Create(ctx, pod, binding, &client.SubResourceCreateOptions{}) + Expect((err)).NotTo(HaveOccurred()) + + By("Asserting the pod is bound") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Spec.NodeName).To(Equal(node.Name)) + }) + + It("should be able to approve CSRs", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + err = cl.SubResource("approval").Update(ctx, csr, &client.SubResourceUpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to approve CSRs using Patch", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + patch := client.MergeFrom(csr.DeepCopy()) + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + err = cl.SubResource("approval").Patch(ctx, csr, patch, &client.SubResourcePatchOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to update the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresource") + replicaCount := *dep.Spec.Replicas + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: replicaCount}} + err = cl.SubResource("scale").Update(ctx, dep, client.WithSubResourceBody(scale), &client.SubResourceUpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + + It("should be able to patch the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + patch := client.MergeFrom(&autoscalingv1.Scale{}) + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: replicaCount}} + err = cl.SubResource("scale").Patch(ctx, dep, patch, client.WithSubResourceBody(scale), &client.SubResourcePatchOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + + It("should be able to apply the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + replicaCount := *dep.Spec.Replicas + 1 + + By("Applying the scale subresurce") + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "foo") + Expect(err).NotTo(HaveOccurred()) + scale := autoscaling1applyconfigurations.Scale(). + WithSpec(autoscaling1applyconfigurations.ScaleSpec().WithReplicas(replicaCount)) + err = cl.SubResource("scale").Apply(ctx, deploymentAC, + &client.SubResourceApplyOptions{SubResourceBody: scale}, + client.FieldOwner("foo"), + client.ForceOwnership, + ) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + }) + + Context("with unstructured objects", func() { + It("should be able to read the Scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = appsv1.SchemeGroupVersion.String() + dep.Kind = reflect.TypeOf(dep).Elem().Name() + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("reading the scale subresource") + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + err = cl.SubResource("scale").Get(ctx, depUnstructured, scale) + Expect(err).NotTo(HaveOccurred()) + + val, found, err := unstructured.NestedInt64(scale.UnstructuredContent(), "spec", "replicas") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(int32(val)).To(Equal(*dep.Spec.Replicas)) + }) + It("should be able to create ServiceAccount tokens", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the serviceAccount") + _, err = clientset.CoreV1().ServiceAccounts(serviceAccount.Namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) + Expect((err)).NotTo(HaveOccurred()) + + serviceAccount.APIVersion = "v1" + serviceAccount.Kind = "ServiceAccount" + serviceAccountUnstructured, err := toUnstructured(serviceAccount) + Expect(err).NotTo(HaveOccurred()) + + token := &unstructured.Unstructured{} + token.SetAPIVersion("authentication.k8s.io/v1") + token.SetKind("TokenRequest") + err = cl.SubResource("token").Create(ctx, serviceAccountUnstructured, token) + Expect(err).NotTo(HaveOccurred()) + Expect(token.GetAPIVersion()).To(Equal("authentication.k8s.io/v1")) + Expect(token.GetKind()).To(Equal("TokenRequest")) + + val, found, err := unstructured.NestedString(token.UnstructuredContent(), "status", "token") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(val).NotTo(Equal("")) + }) + + It("should be able to create Pod evictions", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + pod.APIVersion = "v1" + pod.Kind = "Pod" + podUnstructured, err := toUnstructured(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the eviction") + eviction := &unstructured.Unstructured{} + eviction.SetAPIVersion("policy/v1") + eviction.SetKind("Eviction") + err = unstructured.SetNestedField(eviction.UnstructuredContent(), int64(0), "deleteOptions", "gracePeriodSeconds") + Expect(err).NotTo(HaveOccurred()) + err = cl.SubResource("eviction").Create(ctx, podUnstructured, eviction) + Expect(err).NotTo(HaveOccurred()) + Expect(eviction.GetAPIVersion()).To(Equal("policy/v1")) + Expect(eviction.GetKind()).To(Equal("Eviction")) + + By("Asserting the pod is gone") + _, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should be able to create Pod bindings", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + pod.APIVersion = "v1" + pod.Kind = "Pod" + podUnstructured, err := toUnstructured(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the binding") + binding := &unstructured.Unstructured{} + binding.SetAPIVersion("v1") + binding.SetKind("Binding") + err = unstructured.SetNestedField(binding.UnstructuredContent(), node.Name, "target", "name") + Expect(err).NotTo(HaveOccurred()) + + err = cl.SubResource("binding").Create(ctx, podUnstructured, binding) + Expect((err)).NotTo(HaveOccurred()) + Expect(binding.GetAPIVersion()).To(Equal("v1")) + Expect(binding.GetKind()).To(Equal("Binding")) + + By("Asserting the pod is bound") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Spec.NodeName).To(Equal(node.Name)) + }) + + It("should be able to approve CSRs", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + csr.APIVersion = "certificates.k8s.io/v1" + csr.Kind = "CertificateSigningRequest" + csrUnstructured, err := toUnstructured(csr) + Expect(err).NotTo(HaveOccurred()) + + err = cl.SubResource("approval").Update(ctx, csrUnstructured) + Expect(err).NotTo(HaveOccurred()) + Expect(csrUnstructured.GetAPIVersion()).To(Equal("certificates.k8s.io/v1")) + Expect(csrUnstructured.GetKind()).To(Equal("CertificateSigningRequest")) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to approve CSRs using Patch", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + patch := client.MergeFrom(csr.DeepCopy()) + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + csr.APIVersion = "certificates.k8s.io/v1" + csr.Kind = "CertificateSigningRequest" + csrUnstructured, err := toUnstructured(csr) + Expect(err).NotTo(HaveOccurred()) + + err = cl.SubResource("approval").Patch(ctx, csrUnstructured, patch) + Expect(err).NotTo(HaveOccurred()) + Expect(csrUnstructured.GetAPIVersion()).To(Equal("certificates.k8s.io/v1")) + Expect(csrUnstructured.GetKind()).To(Equal("CertificateSigningRequest")) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to update the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = appsv1.SchemeGroupVersion.String() + dep.Kind = "Deployment" //nolint:goconst + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + Expect(unstructured.SetNestedField(scale.Object, int64(replicaCount), "spec", "replicas")).NotTo(HaveOccurred()) + err = cl.SubResource("scale").Update(ctx, depUnstructured, client.WithSubResourceBody(scale)) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.GetAPIVersion()).To(Equal("autoscaling/v1")) + Expect(scale.GetKind()).To(Equal("Scale")) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + + It("should be able to patch the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = "apps/v1" + dep.Kind = "Deployment" + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + patch := client.MergeFrom(scale.DeepCopy()) + Expect(unstructured.SetNestedField(scale.Object, int64(replicaCount), "spec", "replicas")).NotTo(HaveOccurred()) + err = cl.SubResource("scale").Patch(ctx, depUnstructured, patch, client.WithSubResourceBody(scale)) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.GetAPIVersion()).To(Equal("autoscaling/v1")) + Expect(scale.GetKind()).To(Equal("Scale")) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + + It("should be able to apply the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = "apps/v1" + dep.Kind = "Deployment" + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + 1 + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + Expect(unstructured.SetNestedField(scale.Object, int64(replicaCount), "spec", "replicas")).NotTo(HaveOccurred()) + err = cl.SubResource("scale").Apply(ctx, + client.ApplyConfigurationFromUnstructured(depUnstructured), + &client.SubResourceApplyOptions{SubResourceBody: client.ApplyConfigurationFromUnstructured(scale)}, + client.FieldOwner("foo"), + client.ForceOwnership, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.GetAPIVersion()).To(Equal("autoscaling/v1")) + Expect(scale.GetKind()).To(Equal("Scale")) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) }) }) + }) Describe("StatusClient", func() { Context("with structured objects", func() { - It("should update status of an existing object", func() { + It("should update status of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -719,7 +1456,7 @@ var _ = Describe("Client", func() { By("updating the status of Deployment") dep.Status.Replicas = 1 - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status") @@ -729,7 +1466,7 @@ var _ = Describe("Client", func() { Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) }) - It("should update status and preserve type information", func() { + It("should update status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -741,14 +1478,14 @@ var _ = Describe("Client", func() { By("updating the status of Deployment") dep.SetGroupVersionKind(depGvk) dep.Status.Replicas = 1 - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) }) - It("should patch status and preserve type information", func() { + It("should patch status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -761,14 +1498,37 @@ var _ = Describe("Client", func() { dep.SetGroupVersionKind(depGvk) depPatch := client.MergeFrom(dep.DeepCopy()) dep.Status.Replicas = 1 - err = cl.Status().Patch(context.TODO(), dep, depPatch) + err = cl.Status().Patch(ctx, dep, depPatch) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) }) - It("should not update spec of an existing object", func() { + It("should apply status", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("initially creating a Deployment") + dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Status.Replicas).To(BeEquivalentTo(0)) + + By("applying the status of Deployment") + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "foo") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(1)), + }) + Expect(cl.Status().Apply(ctx, deploymentAC, client.FieldOwner("foo"))).To(Succeed()) + + dep, err = clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Status.Replicas).To(BeEquivalentTo(1)) + }) + + It("should not update spec of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -781,7 +1541,7 @@ var _ = Describe("Client", func() { var rc int32 = 1 dep.Status.Replicas = 1 dep.Spec.Replicas = &rc - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status and unchanged spec") @@ -792,7 +1552,7 @@ var _ = Describe("Client", func() { Expect(*actual.Spec.Replicas).To(BeEquivalentTo(replicaCount)) }) - It("should update an existing object non-namespace object", func() { + It("should update an existing object non-namespace object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -802,7 +1562,7 @@ var _ = Describe("Client", func() { By("updating status of the object") node.Status.Phase = corev1.NodeRunning - err = cl.Status().Update(context.TODO(), node) + err = cl.Status().Update(ctx, node) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -812,17 +1572,17 @@ var _ = Describe("Client", func() { Expect(actual.Status.Phase).To(Equal(corev1.NodeRunning)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("updating status of a non-existent object") - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).To(HaveOccurred()) }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -835,7 +1595,7 @@ var _ = Describe("Client", func() { By("updating status of the Deployment") dep.Status.Replicas = 1 - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -850,7 +1610,7 @@ var _ = Describe("Client", func() { }) Context("with unstructured objects", func() { - It("should update status of an existing object", func() { + It("should update status of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -863,7 +1623,7 @@ var _ = Describe("Client", func() { u := &unstructured.Unstructured{} dep.Status.Replicas = 1 Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status") @@ -873,7 +1633,7 @@ var _ = Describe("Client", func() { Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) }) - It("should update status and preserve type information", func() { + It("should update status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -886,14 +1646,14 @@ var _ = Describe("Client", func() { u := &unstructured.Unstructured{} dep.Status.Replicas = 1 Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(u.GroupVersionKind()).To(Equal(depGvk)) }) - It("should patch status and preserve type information", func() { + It("should patch status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -907,7 +1667,7 @@ var _ = Describe("Client", func() { depPatch := client.MergeFrom(dep.DeepCopy()) dep.Status.Replicas = 1 Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Patch(context.TODO(), u, depPatch) + err = cl.Status().Patch(ctx, u, depPatch) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") @@ -920,7 +1680,35 @@ var _ = Describe("Client", func() { Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) }) - It("should not update spec of an existing object", func() { + It("should apply status and preserve type information", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("initially creating a Deployment") + dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Status.Replicas).To(BeEquivalentTo(0)) + + By("applying the status of Deployment") + dep.Status.Replicas = 1 + dep.ManagedFields = nil // Must be unset in SSA requests + u := &unstructured.Unstructured{} + Expect(scheme.Convert(dep, u, nil)).To(Succeed()) + err = cl.Status().Apply(ctx, client.ApplyConfigurationFromUnstructured(u), client.FieldOwner("foo")) + Expect(err).NotTo(HaveOccurred()) + + By("validating updated Deployment has type information") + Expect(u.GroupVersionKind()).To(Equal(depGvk)) + + By("validating patched Deployment has new status") + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) + }) + + It("should not update spec of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -935,7 +1723,7 @@ var _ = Describe("Client", func() { dep.Status.Replicas = 1 dep.Spec.Replicas = &rc Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status and unchanged spec") @@ -946,7 +1734,7 @@ var _ = Describe("Client", func() { Expect(*actual.Spec.Replicas).To(BeEquivalentTo(replicaCount)) }) - It("should update an existing object non-namespace object", func() { + It("should update an existing object non-namespace object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -958,7 +1746,7 @@ var _ = Describe("Client", func() { u := &unstructured.Unstructured{} node.Status.Phase = corev1.NodeRunning Expect(scheme.Convert(node, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -968,7 +1756,7 @@ var _ = Describe("Client", func() { Expect(actual.Status.Phase).To(Equal(corev1.NodeRunning)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -976,7 +1764,7 @@ var _ = Describe("Client", func() { By("updating status of a non-existent object") u := &unstructured.Unstructured{} Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).To(HaveOccurred()) }) @@ -991,15 +1779,15 @@ var _ = Describe("Client", func() { }) Context("with metadata objects", func() { - It("should fail to update with an error", func() { + It("should fail to update with an error", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) obj := metaOnlyFromObj(dep, scheme) - Expect(cl.Status().Update(context.TODO(), obj)).NotTo(Succeed()) + Expect(cl.Status().Update(ctx, obj)).NotTo(Succeed()) }) - It("should patch status and preserve type information", func() { + It("should patch status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1012,7 +1800,7 @@ var _ = Describe("Client", func() { objPatch := client.MergeFrom(metaOnlyFromObj(dep, scheme)) dep.Annotations = map[string]string{"some-new-annotation": "some-new-value"} obj := metaOnlyFromObj(dep, scheme) - err = cl.Status().Patch(context.TODO(), obj, objPatch) + err = cl.Status().Patch(ctx, obj, objPatch) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") @@ -1029,7 +1817,7 @@ var _ = Describe("Client", func() { Describe("Delete", func() { Context("with structured objects", func() { - It("should delete an existing object from a go struct", func() { + It("should delete an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1040,7 +1828,7 @@ var _ = Describe("Client", func() { By("deleting the Deployment") depName := dep.Name - err = cl.Delete(context.TODO(), dep) + err = cl.Delete(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1048,7 +1836,7 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) }) - It("should delete an existing object non-namespace object from a go struct", func() { + It("should delete an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1059,7 +1847,7 @@ var _ = Describe("Client", func() { By("deleting the Node") nodeName := node.Name - err = cl.Delete(context.TODO(), node) + err = cl.Delete(ctx, node) Expect(err).NotTo(HaveOccurred()) By("validating the Node no longer exists") @@ -1067,13 +1855,13 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("Deleting node before it is ever created") - err = cl.Delete(context.TODO(), node) + err = cl.Delete(ctx, node) Expect(err).To(HaveOccurred()) }) @@ -1081,7 +1869,7 @@ var _ = Describe("Client", func() { }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -1093,7 +1881,7 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) By("deleting the Deployment fails") - err = cl.Delete(context.TODO(), dep) + err = cl.Delete(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -1102,7 +1890,7 @@ var _ = Describe("Client", func() { }) - It("should delete a collection of objects", func() { + It("should delete a collection of objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1110,7 +1898,7 @@ var _ = Describe("Client", func() { By("initially creating two Deployments") dep2 := dep.DeepCopy() - dep2.Name = dep2.Name + "-2" + dep2.Name += "-2" dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1121,7 +1909,7 @@ var _ = Describe("Client", func() { dep2Name := dep2.Name By("deleting Deployments") - err = cl.DeleteAllOf(context.TODO(), dep, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) + err = cl.DeleteAllOf(ctx, dep, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1132,7 +1920,7 @@ var _ = Describe("Client", func() { }) }) Context("with unstructured objects", func() { - It("should delete an existing object from a go struct", func() { + It("should delete an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1150,7 +1938,7 @@ var _ = Describe("Client", func() { Kind: "Deployment", Version: "v1", }) - err = cl.Delete(context.TODO(), u) + err = cl.Delete(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1158,7 +1946,7 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) }) - It("should delete an existing object non-namespace object from a go struct", func() { + It("should delete an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1176,7 +1964,7 @@ var _ = Describe("Client", func() { Kind: "Node", Version: "v1", }) - err = cl.Delete(context.TODO(), u) + err = cl.Delete(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating the Node no longer exists") @@ -1184,7 +1972,7 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1197,11 +1985,11 @@ var _ = Describe("Client", func() { Kind: "Node", Version: "v1", }) - err = cl.Delete(context.TODO(), node) + err = cl.Delete(ctx, node) Expect(err).To(HaveOccurred()) }) - It("should delete a collection of object", func() { + It("should delete a collection of object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1209,7 +1997,7 @@ var _ = Describe("Client", func() { By("initially creating two Deployments") dep2 := dep.DeepCopy() - dep2.Name = dep2.Name + "-2" + dep2.Name += "-2" dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1227,7 +2015,7 @@ var _ = Describe("Client", func() { Kind: "Deployment", Version: "v1", }) - err = cl.DeleteAllOf(context.TODO(), u, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) + err = cl.DeleteAllOf(ctx, u, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1238,7 +2026,7 @@ var _ = Describe("Client", func() { }) }) Context("with metadata objects", func() { - It("should delete an existing object from a go struct", func() { + It("should delete an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1249,7 +2037,7 @@ var _ = Describe("Client", func() { By("deleting the Deployment") metaObj := metaOnlyFromObj(dep, scheme) - err = cl.Delete(context.TODO(), metaObj) + err = cl.Delete(ctx, metaObj) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1257,7 +2045,7 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) }) - It("should delete an existing object non-namespace object from a go struct", func() { + It("should delete an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1268,7 +2056,7 @@ var _ = Describe("Client", func() { By("deleting the Node") metaObj := metaOnlyFromObj(node, scheme) - err = cl.Delete(context.TODO(), metaObj) + err = cl.Delete(ctx, metaObj) Expect(err).NotTo(HaveOccurred()) By("validating the Node no longer exists") @@ -1276,18 +2064,18 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("Deleting node before it is ever created") metaObj := metaOnlyFromObj(node, scheme) - err = cl.Delete(context.TODO(), metaObj) + err = cl.Delete(ctx, metaObj) Expect(err).To(HaveOccurred()) }) - It("should delete a collection of object", func() { + It("should delete a collection of object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1295,7 +2083,7 @@ var _ = Describe("Client", func() { By("initially creating two Deployments") dep2 := dep.DeepCopy() - dep2.Name = dep2.Name + "-2" + dep2.Name += "-2" dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1307,7 +2095,7 @@ var _ = Describe("Client", func() { By("deleting Deployments") metaObj := metaOnlyFromObj(dep, scheme) - err = cl.DeleteAllOf(context.TODO(), metaObj, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) + err = cl.DeleteAllOf(ctx, metaObj, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1321,7 +2109,7 @@ var _ = Describe("Client", func() { Describe("Get", func() { Context("with structured objects", func() { - It("should fetch an existing object for a go struct", func() { + It("should fetch an existing object for a go struct", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1333,7 +2121,7 @@ var _ = Describe("Client", func() { By("fetching the created Deployment") var actual appsv1.Deployment key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) @@ -1341,7 +2129,7 @@ var _ = Describe("Client", func() { Expect(dep).To(Equal(&actual)) }) - It("should fetch an existing non-namespace object for a go struct", func() { + It("should fetch an existing non-namespace object for a go struct", func(ctx SpecContext) { By("first creating the object") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1353,14 +2141,14 @@ var _ = Describe("Client", func() { By("retrieving node through client") var actual corev1.Node key := client.ObjectKey{Namespace: ns, Name: node.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(node).To(Equal(&actual)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1368,7 +2156,7 @@ var _ = Describe("Client", func() { By("fetching object that has not been created yet") key := client.ObjectKey{Namespace: ns, Name: dep.Name} var actual appsv1.Deployment - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).To(HaveOccurred()) }) @@ -1376,7 +2164,7 @@ var _ = Describe("Client", func() { }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1390,7 +2178,7 @@ var _ = Describe("Client", func() { By("fetching the created Deployment fails") var actual appsv1.Deployment key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -1402,8 +2190,7 @@ var _ = Describe("Client", func() { // Test this with an integrated type and a CRD to make sure it covers both proto // and json deserialization. for idx, object := range []client.Object{&corev1.ConfigMap{}, &pkg.ChaosPod{}} { - idx, object := idx, object - It(fmt.Sprintf("should not retain any data in the obj variable that is not on the server for %T", object), func() { + It(fmt.Sprintf("should not retain any data in the obj variable that is not on the server for %T", object), func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1428,7 +2215,7 @@ var _ = Describe("Client", func() { }) Context("with unstructured objects", func() { - It("should fetch an existing object", func() { + It("should fetch an existing object", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1449,15 +2236,16 @@ var _ = Describe("Client", func() { Version: "v1", }) key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) By("validating the fetched Deployment equals the created one") - Expect(u).To(Equal(&actual)) + unstructured.RemoveNestedField(actual.Object, "spec", "template", "metadata", "creationTimestamp") + Expect(u).To(BeComparableTo(&actual)) }) - It("should fetch an existing non-namespace object", func() { + It("should fetch an existing non-namespace object", func(ctx SpecContext) { By("first creating the Node") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1478,7 +2266,7 @@ var _ = Describe("Client", func() { Version: "v1", }) key := client.ObjectKey{Namespace: ns, Name: node.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) @@ -1486,7 +2274,7 @@ var _ = Describe("Client", func() { Expect(u).To(Equal(&actual)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1494,11 +2282,11 @@ var _ = Describe("Client", func() { By("fetching object that has not been created yet") key := client.ObjectKey{Namespace: ns, Name: dep.Name} u := &unstructured.Unstructured{} - err = cl.Get(context.TODO(), key, u) + err = cl.Get(ctx, key, u) Expect(err).To(HaveOccurred()) }) - It("should not retain any data in the obj variable that is not on the server", func() { + It("should not retain any data in the obj variable that is not on the server", func(ctx SpecContext) { object := &unstructured.Unstructured{} cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1523,7 +2311,7 @@ var _ = Describe("Client", func() { }) }) Context("with metadata objects", func() { - It("should fetch an existing object for a go struct", func() { + It("should fetch an existing object for a go struct", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1541,7 +2329,7 @@ var _ = Describe("Client", func() { } actual.SetGroupVersionKind(gvk) key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) @@ -1552,7 +2340,7 @@ var _ = Describe("Client", func() { Expect(metaOnlyFromObj(dep, scheme)).To(Equal(&actual)) }) - It("should fetch an existing non-namespace object for a go struct", func() { + It("should fetch an existing non-namespace object for a go struct", func(ctx SpecContext) { By("first creating the object") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1568,14 +2356,14 @@ var _ = Describe("Client", func() { Kind: "Node", }) key := client.ObjectKey{Namespace: ns, Name: node.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(metaOnlyFromObj(node, scheme)).To(Equal(&actual)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1588,7 +2376,7 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "Deployment", }) - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).To(HaveOccurred()) }) @@ -1600,7 +2388,7 @@ var _ = Describe("Client", func() { }) - It("should not retain any data in the obj variable that is not on the server", func() { + It("should not retain any data in the obj variable that is not on the server", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1625,7 +2413,7 @@ var _ = Describe("Client", func() { Describe("List", func() { Context("with structured objects", func() { - It("should fetch collection of objects", func() { + It("should fetch collection of objects", func(ctx SpecContext) { By("creating an initial object") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1635,7 +2423,7 @@ var _ = Describe("Client", func() { By("listing all objects of that type in the cluster") deps := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), deps)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, deps)).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) hasDep := false @@ -1646,9 +2434,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - }, serverSideTimeoutSeconds) + }) - It("should fetch unstructured collection of objects", func() { + It("should fetch unstructured collection of objects", func(ctx SpecContext) { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1663,7 +2451,7 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps) + err = cl.List(ctx, deps) Expect(err).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) @@ -1680,9 +2468,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - }, serverSideTimeoutSeconds) + }) - It("should fetch unstructured collection of objects, even if scheme is empty", func() { + It("should fetch unstructured collection of objects, even if scheme is empty", func(ctx SpecContext) { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1697,7 +2485,7 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps) + err = cl.List(ctx, deps) Expect(err).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) @@ -1709,22 +2497,22 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - }, serverSideTimeoutSeconds) + }) - It("should return an empty list if there are no matching objects", func() { + It("should return an empty list if there are no matching objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) By("listing all Deployments in the cluster") deps := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), deps)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, deps)).NotTo(HaveOccurred()) By("validating no Deployments are returned") Expect(deps.Items).To(BeEmpty()) - }, serverSideTimeoutSeconds) + }) // TODO(seans): get label selector test working - It("should filter results by label selector", func() { + It("should filter results by label selector", func(ctx SpecContext) { By("creating a Deployment with the app=frontend label") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -1771,7 +2559,7 @@ var _ = Describe("Client", func() { By("listing all Deployments with label app=backend") deps := &appsv1.DeploymentList{} labels := map[string]string{"app": "backend"} - err = cl.List(context.Background(), deps, client.MatchingLabels(labels)) + err = cl.List(ctx, deps, client.MatchingLabels(labels)) Expect(err).NotTo(HaveOccurred()) By("only the Deployment with the backend label is returned") @@ -1782,9 +2570,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) - }, serverSideTimeoutSeconds) + }) - It("should filter results by namespace selector", func() { + It("should filter results by namespace selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-1") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -1828,7 +2616,7 @@ var _ = Describe("Client", func() { By("listing all Deployments in test-namespace-1") deps := &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, client.InNamespace("test-namespace-1")) + err = cl.List(ctx, deps, client.InNamespace("test-namespace-1")) Expect(err).NotTo(HaveOccurred()) By("only the Deployment in test-namespace-1 is returned") @@ -1841,9 +2629,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depBackend, "test-namespace-2") deleteNamespace(ctx, tns1) deleteNamespace(ctx, tns2) - }, serverSideTimeoutSeconds) + }) - It("should filter results by field selector", func() { + It("should filter results by field selector", func(ctx SpecContext) { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -1881,7 +2669,7 @@ var _ = Describe("Client", func() { By("listing all Deployments with field metadata.name=deployment-backend") deps := &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.MatchingFields{"metadata.name": "deployment-backend"}) Expect(err).NotTo(HaveOccurred()) @@ -1893,9 +2681,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) - }, serverSideTimeoutSeconds) + }) - It("should filter results by namespace selector and label selector", func() { + It("should filter results by namespace selector and label selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-3 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-3"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -1968,7 +2756,7 @@ var _ = Describe("Client", func() { By("listing all Deployments in test-namespace-3 with label app=frontend") deps := &appsv1.DeploymentList{} labels := map[string]string{"app": "frontend"} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.InNamespace("test-namespace-3"), client.MatchingLabels(labels), ) @@ -1986,9 +2774,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend4, "test-namespace-4") deleteNamespace(ctx, tns3) deleteNamespace(ctx, tns4) - }, serverSideTimeoutSeconds) + }) - It("should filter results using limit and continue options", func() { + It("should filter results using limit and continue options", func(ctx SpecContext) { makeDeployment := func(suffix string) *appsv1.Deployment { return &appsv1.Deployment{ @@ -2033,7 +2821,7 @@ var _ = Describe("Client", func() { By("listing 1 deployment when limit=1 is used") deps := &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.Limit(1), ) Expect(err).NotTo(HaveOccurred()) @@ -2046,7 +2834,7 @@ var _ = Describe("Client", func() { By("listing the next deployment when previous continuation token is used and limit=1") deps = &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.Limit(1), client.Continue(continueToken), ) @@ -2060,7 +2848,7 @@ var _ = Describe("Client", func() { By("listing the 2 remaining deployments when previous continuation token is used without a limit") deps = &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.Continue(continueToken), ) Expect(err).NotTo(HaveOccurred()) @@ -2069,7 +2857,7 @@ var _ = Describe("Client", func() { Expect(deps.Continue).To(BeEmpty()) Expect(deps.Items[0].Name).To(Equal(dep3.Name)) Expect(deps.Items[1].Name).To(Equal(dep4.Name)) - }, serverSideTimeoutSeconds) + }) PIt("should fail if the object doesn't have meta", func() { @@ -2085,7 +2873,7 @@ var _ = Describe("Client", func() { }) Context("with unstructured objects", func() { - It("should fetch collection of objects", func() { + It("should fetch collection of objects", func(ctx SpecContext) { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2100,7 +2888,7 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps) + err = cl.List(ctx, deps) Expect(err).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) @@ -2112,9 +2900,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - }, serverSideTimeoutSeconds) + }) - It("should return an empty list if there are no matching objects", func() { + It("should return an empty list if there are no matching objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -2125,13 +2913,13 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", Version: "v1", }) - Expect(cl.List(context.Background(), deps)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, deps)).NotTo(HaveOccurred()) By("validating no Deployments are returned") Expect(deps.Items).To(BeEmpty()) - }, serverSideTimeoutSeconds) + }) - It("should filter results by namespace selector", func() { + It("should filter results by namespace selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-5") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-5"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -2180,7 +2968,7 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps, client.InNamespace("test-namespace-5")) + err = cl.List(ctx, deps, client.InNamespace("test-namespace-5")) Expect(err).NotTo(HaveOccurred()) By("only the Deployment in test-namespace-5 is returned") @@ -2193,9 +2981,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depBackend, "test-namespace-6") deleteNamespace(ctx, tns1) deleteNamespace(ctx, tns2) - }, serverSideTimeoutSeconds) + }) - It("should filter results by field selector", func() { + It("should filter results by field selector", func(ctx SpecContext) { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -2238,7 +3026,7 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.MatchingFields{"metadata.name": "deployment-backend"}) Expect(err).NotTo(HaveOccurred()) @@ -2250,9 +3038,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) - }, serverSideTimeoutSeconds) + }) - It("should filter results by namespace selector and label selector", func() { + It("should filter results by namespace selector and label selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-7 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-7"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -2330,7 +3118,7 @@ var _ = Describe("Client", func() { Version: "v1", }) labels := map[string]string{"app": "frontend"} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.InNamespace("test-namespace-7"), client.MatchingLabels(labels)) Expect(err).NotTo(HaveOccurred()) @@ -2346,7 +3134,7 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend4, "test-namespace-8") deleteNamespace(ctx, tns3) deleteNamespace(ctx, tns4) - }, serverSideTimeoutSeconds) + }) PIt("should fail if the object doesn't have meta", func() { @@ -2357,7 +3145,7 @@ var _ = Describe("Client", func() { }) }) Context("with metadata objects", func() { - It("should fetch collection of objects", func() { + It("should fetch collection of objects", func(ctx SpecContext) { By("creating an initial object") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2373,7 +3161,7 @@ var _ = Describe("Client", func() { } metaList := &metav1.PartialObjectMetadataList{} metaList.SetGroupVersionKind(gvk) - Expect(cl.List(context.Background(), metaList)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, metaList)).NotTo(HaveOccurred()) By("validating that the list GVK has been preserved") Expect(metaList.GroupVersionKind()).To(Equal(gvk)) @@ -2394,9 +3182,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - }, serverSideTimeoutSeconds) + }) - It("should return an empty list if there are no matching objects", func() { + It("should return an empty list if there are no matching objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -2407,14 +3195,14 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "DeploymentList", }) - Expect(cl.List(context.Background(), metaList)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, metaList)).NotTo(HaveOccurred()) By("validating no Deployments are returned") Expect(metaList.Items).To(BeEmpty()) - }, serverSideTimeoutSeconds) + }) // TODO(seans): get label selector test working - It("should filter results by label selector", func() { + It("should filter results by label selector", func(ctx SpecContext) { By("creating a Deployment with the app=frontend label") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -2466,7 +3254,7 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", }) labels := map[string]string{"app": "backend"} - err = cl.List(context.Background(), metaList, client.MatchingLabels(labels)) + err = cl.List(ctx, metaList, client.MatchingLabels(labels)) Expect(err).NotTo(HaveOccurred()) By("only the Deployment with the backend label is returned") @@ -2477,9 +3265,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) - }, serverSideTimeoutSeconds) + }) - It("should filter results by namespace selector", func() { + It("should filter results by namespace selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-1") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -2528,7 +3316,7 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, client.InNamespace("test-namespace-1")) + err = cl.List(ctx, metaList, client.InNamespace("test-namespace-1")) Expect(err).NotTo(HaveOccurred()) By("only the Deployment in test-namespace-1 is returned") @@ -2541,9 +3329,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depBackend, "test-namespace-2") deleteNamespace(ctx, tns1) deleteNamespace(ctx, tns2) - }, serverSideTimeoutSeconds) + }) - It("should filter results by field selector", func() { + It("should filter results by field selector", func(ctx SpecContext) { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -2586,7 +3374,7 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.MatchingFields{"metadata.name": "deployment-backend"}) Expect(err).NotTo(HaveOccurred()) @@ -2598,9 +3386,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) - }, serverSideTimeoutSeconds) + }) - It("should filter results by namespace selector and label selector", func() { + It("should filter results by namespace selector and label selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-3 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-3"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -2678,7 +3466,7 @@ var _ = Describe("Client", func() { Kind: "DeploymentList", }) labels := map[string]string{"app": "frontend"} - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.InNamespace("test-namespace-3"), client.MatchingLabels(labels), ) @@ -2696,10 +3484,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend4, "test-namespace-4") deleteNamespace(ctx, tns3) deleteNamespace(ctx, tns4) - }, serverSideTimeoutSeconds) - - It("should filter results using limit and continue options", func() { + }) + It("should filter results using limit and continue options", func(ctx SpecContext) { makeDeployment := func(suffix string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -2748,7 +3535,7 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.Limit(1), ) Expect(err).NotTo(HaveOccurred()) @@ -2766,7 +3553,7 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.Limit(1), client.Continue(continueToken), ) @@ -2785,7 +3572,7 @@ var _ = Describe("Client", func() { Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.Continue(continueToken), ) Expect(err).NotTo(HaveOccurred()) @@ -2794,7 +3581,7 @@ var _ = Describe("Client", func() { Expect(metaList.Continue).To(BeEmpty()) Expect(metaList.Items[0].Name).To(Equal(dep3.Name)) Expect(metaList.Items[1].Name).To(Equal(dep4.Name)) - }, serverSideTimeoutSeconds) + }) PIt("should fail if the object doesn't have meta", func() { @@ -2913,6 +3700,24 @@ var _ = Describe("Client", func() { }) }) + Describe("GetOptions", func() { + It("should be convertable to metav1.GetOptions", func() { + o := (&client.GetOptions{}).ApplyOptions([]client.GetOption{ + &client.GetOptions{Raw: &metav1.GetOptions{ResourceVersion: "RV0"}}, + }) + mo := o.AsGetOptions() + Expect(mo).NotTo(BeNil()) + Expect(mo.ResourceVersion).To(Equal("RV0")) + }) + + It("should produce empty metav1.GetOptions if nil", func() { + var o *client.GetOptions + Expect(o.AsGetOptions()).To(Equal(&metav1.GetOptions{})) + o = &client.GetOptions{} + Expect(o.AsGetOptions()).To(Equal(&metav1.GetOptions{})) + }) + }) + Describe("ListOptions", func() { It("should be convertable to metav1.ListOptions", func() { lo := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ @@ -3064,27 +3869,26 @@ var _ = Describe("Client", func() { }) }) -var _ = Describe("DelegatingClient", func() { +var _ = Describe("ClientWithCache", func() { Describe("Get", func() { - It("should call cache reader when structured object", func() { + It("should call cache reader when structured object", func(ctx SpecContext) { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) var actual appsv1.Deployment key := client.ObjectKey{Namespace: "ns", Name: "name"} - Expect(dReader.Get(context.TODO(), key, &actual)).To(Succeed()) + Expect(cl.Get(ctx, key, &actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) When("getting unstructured objects", func() { var dep *appsv1.Deployment - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "deployment1", @@ -3101,23 +3905,22 @@ var _ = Describe("DelegatingClient", func() { }, } var err error - dep, err = clientset.AppsV1().Deployments("default").Create(context.Background(), dep, metav1.CreateOptions{}) + dep, err = clientset.AppsV1().Deployments("default").Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { Expect(clientset.AppsV1().Deployments("default").Delete( - context.Background(), + ctx, dep.Name, metav1.DeleteOptions{}, )).To(Succeed()) }) - It("should call client reader when not cached", func() { + It("should call client reader when not cached", func(ctx SpecContext) { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3129,17 +3932,16 @@ var _ = Describe("DelegatingClient", func() { }) actual.SetName(dep.Name) key := client.ObjectKey{Namespace: dep.Namespace, Name: dep.Name} - Expect(dReader.Get(context.TODO(), key, actual)).To(Succeed()) + Expect(cl.Get(ctx, key, actual)).To(Succeed()) Expect(0).To(Equal(cachedReader.Called)) }) - It("should call cache reader when cached", func() { + It("should call cache reader when cached", func(ctx SpecContext) { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, - CacheUnstructured: true, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + Unstructured: true, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3151,34 +3953,32 @@ var _ = Describe("DelegatingClient", func() { }) actual.SetName(dep.Name) key := client.ObjectKey{Namespace: dep.Namespace, Name: dep.Name} - Expect(dReader.Get(context.TODO(), key, actual)).To(Succeed()) + Expect(cl.Get(ctx, key, actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) }) }) Describe("List", func() { - It("should call cache reader when structured object", func() { + It("should call cache reader when structured object", func(ctx SpecContext) { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) var actual appsv1.DeploymentList - Expect(dReader.List(context.Background(), &actual)).To(Succeed()) + Expect(cl.List(ctx, &actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) When("listing unstructured objects", func() { - It("should call client reader when not cached", func() { + It("should call client reader when not cached", func(ctx SpecContext) { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3188,17 +3988,16 @@ var _ = Describe("DelegatingClient", func() { Kind: "DeploymentList", Version: "v1", }) - Expect(dReader.List(context.Background(), actual)).To(Succeed()) + Expect(cl.List(ctx, actual)).To(Succeed()) Expect(0).To(Equal(cachedReader.Called)) }) - It("should call cache reader when cached", func() { + It("should call cache reader when cached", func(ctx SpecContext) { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, - CacheUnstructured: true, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + Unstructured: true, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3208,7 +4007,7 @@ var _ = Describe("DelegatingClient", func() { Kind: "DeploymentList", Version: "v1", }) - Expect(dReader.List(context.Background(), actual)).To(Succeed()) + Expect(cl.List(ctx, actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) }) @@ -3385,11 +4184,37 @@ var _ = Describe("IgnoreNotFound", func() { }) }) +var _ = Describe("IgnoreAlreadyExists", func() { + It("should return nil on a 'AlreadyExists' error", func() { + By("creating a AlreadyExists error") + err := apierrors.NewAlreadyExists(schema.GroupResource{}, "") + + By("returning no error") + Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) + }) + + It("should return the error on a status other than already exists", func() { + By("creating a BadRequest error") + err := apierrors.NewBadRequest("") + + By("returning an error") + Expect(client.IgnoreAlreadyExists(err)).To(HaveOccurred()) + }) + + It("should return the error on a non-status error", func() { + By("creating an fmt error") + err := fmt.Errorf("arbitrary error") + + By("returning an error") + Expect(client.IgnoreAlreadyExists(err)).To(HaveOccurred()) + }) +}) + type fakeReader struct { Called int } -func (f *fakeReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { +func (f *fakeReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { f.Called++ return nil } @@ -3398,3 +4223,26 @@ func (f *fakeReader) List(ctx context.Context, list client.ObjectList, opts ...c f.Called++ return nil } + +type fakeUncachedReader struct { + Called int +} + +func (f *fakeUncachedReader) Get(_ context.Context, _ client.ObjectKey, _ client.Object, opts ...client.GetOption) error { + f.Called++ + return &cache.ErrResourceNotCached{} +} + +func (f *fakeUncachedReader) List(_ context.Context, _ client.ObjectList, _ ...client.ListOption) error { + f.Called++ + return &cache.ErrResourceNotCached{} +} + +func toUnstructured(o client.Object) (*unstructured.Unstructured, error) { + serialized, err := json.Marshal(o) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{} + return u, json.Unmarshal(serialized, u) +} diff --git a/pkg/client/config/config.go b/pkg/client/config/config.go index da87f2bd43..1c39f4d854 100644 --- a/pkg/client/config/config.go +++ b/pkg/client/config/config.go @@ -29,25 +29,42 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) +// KubeconfigFlagName is the name of the kubeconfig flag +const KubeconfigFlagName = "kubeconfig" + var ( kubeconfig string log = logf.RuntimeLog.WithName("client").WithName("config") ) +// init registers the "kubeconfig" flag to the default command line FlagSet. +// TODO: This should be removed, as it potentially leads to redefined flag errors for users, if they already +// have registered the "kubeconfig" flag to the command line FlagSet in other parts of their code. func init() { - // TODO: Fix this to allow double vendoring this library but still register flags on behalf of users - flag.StringVar(&kubeconfig, "kubeconfig", "", - "Paths to a kubeconfig. Only required if out-of-cluster.") + RegisterFlags(flag.CommandLine) +} + +// RegisterFlags registers flag variables to the given FlagSet if not already registered. +// It uses the default command line FlagSet, if none is provided. Currently, it only registers the kubeconfig flag. +func RegisterFlags(fs *flag.FlagSet) { + if fs == nil { + fs = flag.CommandLine + } + if f := fs.Lookup(KubeconfigFlagName); f != nil { + kubeconfig = f.Value.String() + } else { + fs.StringVar(&kubeconfig, KubeconfigFlagName, "", "Paths to a kubeconfig. Only required if out-of-cluster.") + } } // GetConfig creates a *rest.Config for talking to a Kubernetes API server. // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // -// It also applies saner defaults for QPS and burst based on the Kubernetes -// controller manager defaults (20 QPS, 30 burst) +// The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and +// fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. // -// Config precedence +// Config precedence: // // * --kubeconfig flag pointing at a file // @@ -64,10 +81,10 @@ func GetConfig() (*rest.Config, error) { // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // -// It also applies saner defaults for QPS and burst based on the Kubernetes -// controller manager defaults (20 QPS, 30 burst) +// The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and +// fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. // -// Config precedence +// Config precedence: // // * --kubeconfig flag pointing at a file // @@ -81,12 +98,11 @@ func GetConfigWithContext(context string) (*rest.Config, error) { if err != nil { return nil, err } - if cfg.QPS == 0.0 { - cfg.QPS = 20.0 - cfg.Burst = 30.0 + // Disable client-side ratelimer by default, we can rely on + // API priority and fairness + cfg.QPS = -1 } - return cfg, nil } @@ -96,7 +112,7 @@ func GetConfigWithContext(context string) (*rest.Config, error) { var loadInClusterConfig = rest.InClusterConfig // loadConfig loads a REST Config as per the rules specified in GetConfig. -func loadConfig(context string) (*rest.Config, error) { +func loadConfig(context string) (config *rest.Config, configErr error) { // If a flag is specified with the config location, use that if len(kubeconfig) > 0 { return loadConfigWithContext("", &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, context) @@ -106,9 +122,16 @@ func loadConfig(context string) (*rest.Config, error) { // try the in-cluster config. kubeconfigPath := os.Getenv(clientcmd.RecommendedConfigPathEnvVar) if len(kubeconfigPath) == 0 { - if c, err := loadInClusterConfig(); err == nil { + c, err := loadInClusterConfig() + if err == nil { return c, nil } + + defer func() { + if configErr != nil { + log.Error(err, "unable to load in-cluster config") + } + }() } // If the recommended kubeconfig env variable is set, or there @@ -123,7 +146,7 @@ func loadConfig(context string) (*rest.Config, error) { if _, ok := os.LookupEnv("HOME"); !ok { u, err := user.Current() if err != nil { - return nil, fmt.Errorf("could not get current user: %v", err) + return nil, fmt.Errorf("could not get current user: %w", err) } loadingRules.Precedence = append(loadingRules.Precedence, filepath.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName)) } @@ -146,6 +169,9 @@ func loadConfigWithContext(apiServerURL string, loader clientcmd.ClientConfigLoa // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // +// The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and +// fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. +// // Will log an error and exit if there is an error creating the rest.Config. func GetConfigOrDie() *rest.Config { config, err := GetConfig() diff --git a/pkg/client/config/config_suite_test.go b/pkg/client/config/config_suite_test.go index 4d07c03c4b..626613cef4 100644 --- a/pkg/client/config/config_suite_test.go +++ b/pkg/client/config/config_suite_test.go @@ -19,20 +19,18 @@ package config import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestConfig(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Client Config Test Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Client Config Test Suite") } var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) -}, 60) +}) diff --git a/pkg/client/config/config_test.go b/pkg/client/config/config_test.go index ed9761e50f..bbaeb2e2bd 100644 --- a/pkg/client/config/config_test.go +++ b/pkg/client/config/config_test.go @@ -17,12 +17,11 @@ limitations under the License. package config import ( - "io/ioutil" "os" "path/filepath" "strings" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -45,7 +44,7 @@ var _ = Describe("Config", func() { BeforeEach(func() { // create temporary directory for test case var err error - dir, err = ioutil.TempDir("", "cr-test") + dir, err = os.MkdirTemp("", "cr-test") Expect(err).NotTo(HaveOccurred()) // override $HOME/.kube/config @@ -53,7 +52,7 @@ var _ = Describe("Config", func() { }) AfterEach(func() { - os.Unsetenv(clientcmd.RecommendedConfigPathEnvVar) + _ = os.Unsetenv(clientcmd.RecommendedConfigPathEnvVar) kubeconfig = "" clientcmd.RecommendedHomeFile = origRecommendedHomeFile @@ -73,6 +72,7 @@ var _ = Describe("Config", func() { cfg, err := GetConfigWithContext(tc.context) Expect(err).NotTo(HaveOccurred()) Expect(cfg.Host).To(Equal(tc.wantHost)) + Expect(cfg.QPS).To(Equal(float32(-1))) }) } } @@ -83,8 +83,8 @@ var _ = Describe("Config", func() { Expect(err).NotTo(HaveOccurred()) cfg, err := GetConfigWithContext("") - Expect(cfg).To(BeNil()) Expect(err).To(HaveOccurred()) + Expect(cfg).To(BeNil()) }) }) @@ -192,7 +192,7 @@ func setConfigs(tc testCase, dir string) { func createFiles(files map[string]string, dir string) error { for path, data := range files { - if err := ioutil.WriteFile(filepath.Join(dir, path), []byte(data), 0644); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(dir, path), []byte(data), 0644); err != nil { return err } } diff --git a/pkg/client/doc.go b/pkg/client/doc.go index 2965e5fa94..b2e2024942 100644 --- a/pkg/client/doc.go +++ b/pkg/client/doc.go @@ -17,7 +17,7 @@ limitations under the License. // Package client contains functionality for interacting with Kubernetes API // servers. // -// Clients +// # Clients // // Clients are split into two interfaces -- Readers and Writers. Readers // get and list, while writers create, update, and delete. @@ -25,18 +25,18 @@ limitations under the License. // The New function can be used to create a new client that talks directly // to the API server. // -// A common pattern in Kubernetes to read from a cache and write to the API -// server. This pattern is covered by the DelegatingClient type, which can -// be used to have a client whose Reader is different from the Writer. +// It is a common pattern in Kubernetes to read from a cache and write to the API +// server. This pattern is covered by the creating the Client with a Cache. // -// Options +// # Options // // Many client operations in Kubernetes support options. These options are // represented as variadic arguments at the end of a given method call. // For instance, to use a label selector on list, you can call -// err := someReader.List(context.Background(), &podList, client.MatchingLabels{"somelabel": "someval"}) // -// Indexing +// err := someReader.List(context.Background(), &podList, client.MatchingLabels{"somelabel": "someval"}) +// +// # Indexing // // Indexes may be added to caches using a FieldIndexer. This allows you to easily // and efficiently look up objects with certain properties. You can then make diff --git a/pkg/client/dryrun.go b/pkg/client/dryrun.go index ea25ea2530..fb7012200f 100644 --- a/pkg/client/dryrun.go +++ b/pkg/client/dryrun.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ) // NewDryRunClient wraps an existing client and enforces DryRun mode @@ -46,6 +47,16 @@ func (c *dryRunClient) RESTMapper() meta.RESTMapper { return c.client.RESTMapper() } +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (c *dryRunClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.client.GroupVersionKindFor(obj) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (c *dryRunClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.client.IsObjectNamespaced(obj) +} + // Create implements client.Client. func (c *dryRunClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { return c.client.Create(ctx, obj, append(opts, DryRunAll)...) @@ -71,9 +82,13 @@ func (c *dryRunClient) Patch(ctx context.Context, obj Object, patch Patch, opts return c.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...) } +func (c *dryRunClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + return c.client.Apply(ctx, obj, append(opts, DryRunAll)...) +} + // Get implements client.Client. -func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - return c.client.Get(ctx, key, obj) +func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + return c.client.Get(ctx, key, obj, opts...) } // List implements client.Client. @@ -82,25 +97,42 @@ func (c *dryRunClient) List(ctx context.Context, obj ObjectList, opts ...ListOpt } // Status implements client.StatusClient. -func (c *dryRunClient) Status() StatusWriter { - return &dryRunStatusWriter{client: c.client.Status()} +func (c *dryRunClient) Status() SubResourceWriter { + return c.SubResource("status") +} + +// SubResource implements client.SubResourceClient. +func (c *dryRunClient) SubResource(subResource string) SubResourceClient { + return &dryRunSubResourceClient{client: c.client.SubResource(subResource)} } -// ensure dryRunStatusWriter implements client.StatusWriter. -var _ StatusWriter = &dryRunStatusWriter{} +// ensure dryRunSubResourceWriter implements client.SubResourceWriter. +var _ SubResourceWriter = &dryRunSubResourceClient{} -// dryRunStatusWriter is client.StatusWriter that writes status subresource with dryRun mode +// dryRunSubResourceClient is client.SubResourceWriter that writes status subresource with dryRun mode // enforced. -type dryRunStatusWriter struct { - client StatusWriter +type dryRunSubResourceClient struct { + client SubResourceClient +} + +func (sw *dryRunSubResourceClient) Get(ctx context.Context, obj, subResource Object, opts ...SubResourceGetOption) error { + return sw.client.Get(ctx, obj, subResource, opts...) } -// Update implements client.StatusWriter. -func (sw *dryRunStatusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { +func (sw *dryRunSubResourceClient) Create(ctx context.Context, obj, subResource Object, opts ...SubResourceCreateOption) error { + return sw.client.Create(ctx, obj, subResource, append(opts, DryRunAll)...) +} + +// Update implements client.SubResourceWriter. +func (sw *dryRunSubResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { return sw.client.Update(ctx, obj, append(opts, DryRunAll)...) } -// Patch implements client.StatusWriter. -func (sw *dryRunStatusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { +// Patch implements client.SubResourceWriter. +func (sw *dryRunSubResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { return sw.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...) } + +func (sw *dryRunSubResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + return sw.client.Apply(ctx, obj, append(opts, DryRunAll)...) +} diff --git a/pkg/client/dryrun_test.go b/pkg/client/dryrun_test.go index 0a46e5617d..35a9b63869 100644 --- a/pkg/client/dryrun_test.go +++ b/pkg/client/dryrun_test.go @@ -17,17 +17,18 @@ limitations under the License. package client_test import ( - "context" "fmt" "sync/atomic" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + appsv1applyconfigurations "k8s.io/client-go/applyconfigurations/apps/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -37,16 +38,15 @@ var _ = Describe("DryRunClient", func() { var count uint64 = 0 var replicaCount int32 = 2 var ns = "default" - ctx := context.Background() getClient := func() client.Client { - nonDryRunClient, err := client.New(cfg, client.Options{}) + cl, err := client.New(cfg, client.Options{DryRun: ptr.To(true)}) Expect(err).NotTo(HaveOccurred()) - Expect(nonDryRunClient).NotTo(BeNil()) - return client.NewDryRunClient(nonDryRunClient) + Expect(cl).NotTo(BeNil()) + return cl } - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { atomic.AddUint64(&count, 1) dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -71,11 +71,11 @@ var _ = Describe("DryRunClient", func() { Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully Get an object", func() { + It("should successfully Get an object", func(ctx SpecContext) { name := types.NamespacedName{Namespace: ns, Name: dep.Name} result := &appsv1.Deployment{} @@ -83,7 +83,7 @@ var _ = Describe("DryRunClient", func() { Expect(result).To(BeEquivalentTo(dep)) }) - It("should successfully List objects", func() { + It("should successfully List objects", func(ctx SpecContext) { result := &appsv1.DeploymentList{} opts := client.MatchingLabels(dep.Labels) @@ -93,7 +93,7 @@ var _ = Describe("DryRunClient", func() { Expect(result.Items[0]).To(BeEquivalentTo(*dep)) }) - It("should not create an object", func() { + It("should not create an object", func(ctx SpecContext) { newDep := dep.DeepCopy() newDep.Name = "new-deployment" @@ -103,7 +103,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should not create an object with opts", func() { + It("should not create an object with opts", func(ctx SpecContext) { newDep := dep.DeepCopy() newDep.Name = "new-deployment" opts := &client.CreateOptions{DryRun: []string{"Bye", "Pippa"}} @@ -114,7 +114,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should refuse a create request for an invalid object", func() { + It("should refuse a create request for an invalid object", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Spec.Template.Spec.Containers = nil @@ -122,7 +122,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsInvalid(err)).To(BeTrue()) }) - It("should not change objects via update", func() { + It("should not change objects via update", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 @@ -134,7 +134,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via update with opts", func() { + It("should not change objects via update with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 opts := &client.UpdateOptions{DryRun: []string{"Bye", "Pippa"}} @@ -147,7 +147,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should refuse an update request for an invalid change", func() { + It("should refuse an update request for an invalid change", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Spec.Template.Spec.Containers = nil @@ -155,7 +155,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsInvalid(err)).To(BeTrue()) }) - It("should not change objects via patch", func() { + It("should not change objects via patch", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 @@ -167,7 +167,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via patch with opts", func() { + It("should not change objects via patch with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 opts := &client.PatchOptions{DryRun: []string{"Bye", "Pippa"}} @@ -180,7 +180,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not delete objects", func() { + It("should not delete objects", func(ctx SpecContext) { Expect(getClient().Delete(ctx, dep)).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -189,7 +189,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not delete objects with opts", func() { + It("should not delete objects with opts", func(ctx SpecContext) { opts := &client.DeleteOptions{DryRun: []string{"Bye", "Pippa"}} Expect(getClient().Delete(ctx, dep, opts)).NotTo(HaveOccurred()) @@ -200,7 +200,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not delete objects via deleteAllOf", func() { + It("should not delete objects via deleteAllOf", func(ctx SpecContext) { opts := []client.DeleteAllOfOption{client.InNamespace(ns), client.MatchingLabels(dep.Labels)} Expect(getClient().DeleteAllOf(ctx, dep, opts...)).NotTo(HaveOccurred()) @@ -211,7 +211,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via update status", func() { + It("should not change objects via update status", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 @@ -223,10 +223,10 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via update status with opts", func() { + It("should not change objects via update status with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - opts := &client.UpdateOptions{DryRun: []string{"Bye", "Pippa"}} + opts := &client.SubResourceUpdateOptions{UpdateOptions: client.UpdateOptions{DryRun: []string{"Bye", "Pippa"}}} Expect(getClient().Status().Update(ctx, changedDep, opts)).NotTo(HaveOccurred()) @@ -236,7 +236,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via status patch", func() { + It("should not change objects via status patch", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 @@ -248,11 +248,11 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via status patch with opts", func() { + It("should not change objects via status patch with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - opts := &client.PatchOptions{DryRun: []string{"Bye", "Pippa"}} + opts := &client.SubResourcePatchOptions{PatchOptions: client.PatchOptions{DryRun: []string{"Bye", "Pippa"}}} Expect(getClient().Status().Patch(ctx, changedDep, client.MergeFrom(dep), opts)).ToNot(HaveOccurred()) @@ -261,4 +261,36 @@ var _ = Describe("DryRunClient", func() { Expect(actual).NotTo(BeNil()) Expect(actual).To(BeEquivalentTo(dep)) }) + + It("should not change objects via status apply", func(ctx SpecContext) { + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "test-owner") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(99)), + }) + + Expect(getClient().Status().Apply(ctx, deploymentAC, client.FieldOwner("test-owner"))).NotTo(HaveOccurred()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual).To(BeEquivalentTo(dep)) + }) + + It("should not change objects via status apply with opts", func(ctx SpecContext) { + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "test-owner") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(99)), + }) + + opts := &client.SubResourceApplyOptions{ApplyOptions: client.ApplyOptions{DryRun: []string{"Bye", "Pippa"}}} + + Expect(getClient().Status().Apply(ctx, deploymentAC, client.FieldOwner("test-owner"), opts)).NotTo(HaveOccurred()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual).To(BeEquivalentTo(dep)) + }) }) diff --git a/pkg/client/example_test.go b/pkg/client/example_test.go index 1be098bbbb..390dc10143 100644 --- a/pkg/client/example_test.go +++ b/pkg/client/example_test.go @@ -25,11 +25,15 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) var ( @@ -53,7 +57,27 @@ func ExampleNew() { } } -// This example shows how to use the client with typed and unstructured objects to retrieve a objects. +func ExampleNew_suppress_warnings() { + cfg := config.GetConfigOrDie() + // Use a rest.WarningHandlerWithContext that discards warning messages. + cfg.WarningHandlerWithContext = rest.NoWarnings{} + + cl, err := client.New(cfg, client.Options{}) + if err != nil { + fmt.Println("failed to create client") + os.Exit(1) + } + + podList := &corev1.PodList{} + + err = cl.List(context.Background(), podList, client.InNamespace("default")) + if err != nil { + fmt.Printf("failed to list pods in namespace default: %v\n", err) + os.Exit(1) + } +} + +// This example shows how to use the client with typed and unstructured objects to retrieve an object. func ExampleClient_get() { // Using a typed object. pod := &corev1.Pod{} @@ -159,7 +183,7 @@ func ExampleClient_update() { Namespace: "namespace", Name: "name", }, pod) - pod.SetFinalizers(append(pod.GetFinalizers(), "new-finalizer")) + controllerutil.AddFinalizer(pod, "new-finalizer") _ = c.Update(context.Background(), pod) // Using a unstructured object. @@ -173,7 +197,7 @@ func ExampleClient_update() { Namespace: "namespace", Name: "name", }, u) - u.SetFinalizers(append(u.GetFinalizers(), "new-finalizer")) + controllerutil.AddFinalizer(u, "new-finalizer") _ = c.Update(context.Background(), u) } @@ -188,6 +212,18 @@ func ExampleClient_patch() { }, client.RawPatch(types.StrategicMergePatchType, patch)) } +// This example shows how to use the client with unstructured objects to create/patch objects using Server Side Apply, +// "k8s.io/apimachinery/pkg/runtime".DefaultUnstructuredConverter.ToUnstructured is used to convert an object into map[string]any representation, +// which is then set as an "Object" field in *unstructured.Unstructured struct, which implements client.Object. +func ExampleClient_apply() { + // Using a typed object. + configMap := corev1ac.ConfigMap("name", "namespace").WithData(map[string]string{"key": "value"}) + // c is a created client. + u := &unstructured.Unstructured{} + u.Object, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(configMap) + _ = c.Patch(context.Background(), u, client.Apply, client.ForceOwnership, client.FieldOwner("field-owner")) +} + // This example shows how to use the client with typed and unstructured objects to patch objects' status. func ExampleClient_patchStatus() { u := &unstructured.Unstructured{} @@ -230,7 +266,7 @@ func ExampleClient_delete() { _ = c.Delete(context.Background(), u) } -// This example shows how to use the client with typed and unstrucurted objects to delete collections of objects. +// This example shows how to use the client with typed and unstructured objects to delete collections of objects. func ExampleClient_deleteAllOf() { // Using a typed object. // c is a created client. @@ -247,7 +283,7 @@ func ExampleClient_deleteAllOf() { } // This example shows how to set up and consume a field selector over a pod's volumes' secretName field. -func ExampleFieldIndexer_secretName() { +func ExampleFieldIndexer_secretNameNode() { // someIndexer is a FieldIndexer over a Cache _ = someIndexer.IndexField(context.TODO(), &corev1.Pod{}, "spec.volumes.secret.secretName", func(o client.Object) []string { var res []string @@ -261,8 +297,20 @@ func ExampleFieldIndexer_secretName() { return res }) + _ = someIndexer.IndexField(context.TODO(), &corev1.Pod{}, "spec.NodeName", func(o client.Object) []string { + nodeName := o.(*corev1.Pod).Spec.NodeName + if nodeName != "" { + return []string{nodeName} + } + return nil + }) + // elsewhere (e.g. in your reconciler) mySecretName := "someSecret" // derived from the reconcile.Request, for instance + myNode := "master-0" var podsWithSecrets corev1.PodList - _ = c.List(context.Background(), &podsWithSecrets, client.MatchingFields{"spec.volumes.secret.secretName": mySecretName}) + _ = c.List(context.Background(), &podsWithSecrets, client.MatchingFields{ + "spec.volumes.secret.secretName": mySecretName, + "spec.NodeName": myNode, + }) } diff --git a/pkg/client/fake/client.go b/pkg/client/fake/client.go index 1f0efc82bb..62067cb19c 100644 --- a/pkg/client/fake/client.go +++ b/pkg/client/fake/client.go @@ -18,40 +18,83 @@ package fake import ( "context" - "encoding/json" "errors" "fmt" "reflect" - "strconv" "strings" "sync" - + "time" + + /* + Stick with gopkg.in/evanphx/json-patch.v4 here to match + upstream Kubernetes code and avoid breaking changes introduced in v5. + - Kubernetes itself remains on json-patch v4 to avoid compatibility issues + tied to v5’s stricter RFC6902 compliance. + - The fake client code is adapted from client-go’s testing fixture, which also + relies on json-patch v4. + See: + https://github.com/kubernetes/kubernetes/pull/91622 (discussion of why K8s + stays on v4) + https://github.com/kubernetes/kubernetes/pull/120326 (v5.6.0+incompatible + missing a critical fix) + */ + + jsonpatch "gopkg.in/evanphx/json-patch.v4" + appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/managedfields" utilrand "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/watch" + clientgoapplyconfigurations "k8s.io/client-go/applyconfigurations" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/testing" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" "sigs.k8s.io/controller-runtime/pkg/internal/objectutil" ) -type versionedTracker struct { - testing.ObjectTracker - scheme *runtime.Scheme -} - type fakeClient struct { - tracker versionedTracker - scheme *runtime.Scheme - schemeWriteLock sync.Mutex + // trackerWriteLock must be acquired before writing to + // the tracker or performing reads that affect a following + // write. + trackerWriteLock sync.Mutex + tracker versionedTracker + + schemeLock sync.RWMutex + scheme *runtime.Scheme + + restMapper meta.RESTMapper + withStatusSubresource sets.Set[schema.GroupVersionKind] + + // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. + // The inner map maps from index name to IndexerFunc. + indexes map[schema.GroupVersionKind]map[string]client.IndexerFunc + // indexesLock must be held when accessing indexes. + indexesLock sync.RWMutex + + returnManagedFields bool } var _ client.WithWatch = &fakeClient{} @@ -60,25 +103,16 @@ const ( maxNameLength = 63 randomLength = 5 maxGeneratedNameLength = maxNameLength - randomLength + + subResourceScale = "scale" ) // NewFakeClient creates a new fake client for testing. // You can choose to initialize it with a slice of runtime.Object. -// -// Deprecated: Please use NewClientBuilder instead. func NewFakeClient(initObjs ...runtime.Object) client.WithWatch { return NewClientBuilder().WithRuntimeObjects(initObjs...).Build() } -// NewFakeClientWithScheme creates a new fake client with the given scheme -// for testing. -// You can choose to initialize it with a slice of runtime.Object. -// -// Deprecated: Please use NewClientBuilder instead. -func NewFakeClientWithScheme(clientScheme *runtime.Scheme, initObjs ...runtime.Object) client.WithWatch { - return NewClientBuilder().WithScheme(clientScheme).WithRuntimeObjects(initObjs...).Build() -} - // NewClientBuilder returns a new builder to create a fake client. func NewClientBuilder() *ClientBuilder { return &ClientBuilder{} @@ -86,10 +120,21 @@ func NewClientBuilder() *ClientBuilder { // ClientBuilder builds a fake client. type ClientBuilder struct { - scheme *runtime.Scheme - initObject []client.Object - initLists []client.ObjectList - initRuntimeObjects []runtime.Object + scheme *runtime.Scheme + restMapper meta.RESTMapper + initObject []client.Object + initLists []client.ObjectList + initRuntimeObjects []runtime.Object + withStatusSubresource []client.Object + objectTracker testing.ObjectTracker + interceptorFuncs *interceptor.Funcs + typeConverters []managedfields.TypeConverter + returnManagedFields bool + isBuilt bool + + // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. + // The inner map maps from index name to IndexerFunc. + indexes map[schema.GroupVersionKind]map[string]client.IndexerFunc } // WithScheme sets this builder's internal scheme. @@ -99,6 +144,15 @@ func (f *ClientBuilder) WithScheme(scheme *runtime.Scheme) *ClientBuilder { return f } +// WithRESTMapper sets this builder's restMapper. +// The restMapper is directly set as mapper in the Client. This can be used for example +// with a meta.DefaultRESTMapper to provide a static rest mapping. +// If not set, defaults to an empty meta.DefaultRESTMapper. +func (f *ClientBuilder) WithRESTMapper(restMapper meta.RESTMapper) *ClientBuilder { + f.restMapper = restMapper + return f +} + // WithObjects can be optionally used to initialize this fake client with client.Object(s). func (f *ClientBuilder) WithObjects(initObjs ...client.Object) *ClientBuilder { f.initObject = append(f.initObject, initObjs...) @@ -117,13 +171,143 @@ func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *C return f } +// WithObjectTracker can be optionally used to initialize this fake client with testing.ObjectTracker. +// Setting this is incompatible with setting WithTypeConverters, as they are a setting on the +// tracker. +func (f *ClientBuilder) WithObjectTracker(ot testing.ObjectTracker) *ClientBuilder { + f.objectTracker = ot + return f +} + +// WithIndex can be optionally used to register an index with name `field` and indexer `extractValue` +// for API objects of the same GroupVersionKind (GVK) as `obj` in the fake client. +// It can be invoked multiple times, both with objects of the same GVK or different ones. +// Invoking WithIndex twice with the same `field` and GVK (via `obj`) arguments will panic. +// WithIndex retrieves the GVK of `obj` using the scheme registered via WithScheme if +// WithScheme was previously invoked, the default scheme otherwise. +func (f *ClientBuilder) WithIndex(obj runtime.Object, field string, extractValue client.IndexerFunc) *ClientBuilder { + objScheme := f.scheme + if objScheme == nil { + objScheme = scheme.Scheme + } + + gvk, err := apiutil.GVKForObject(obj, objScheme) + if err != nil { + panic(err) + } + + // If this is the first index being registered, we initialize the map storing all the indexes. + if f.indexes == nil { + f.indexes = make(map[schema.GroupVersionKind]map[string]client.IndexerFunc) + } + + // If this is the first index being registered for the GroupVersionKind of `obj`, we initialize + // the map storing the indexes for that GroupVersionKind. + if f.indexes[gvk] == nil { + f.indexes[gvk] = make(map[string]client.IndexerFunc) + } + + if _, fieldAlreadyIndexed := f.indexes[gvk][field]; fieldAlreadyIndexed { + panic(fmt.Errorf("indexer conflict: field %s for GroupVersionKind %v is already indexed", + field, gvk)) + } + + f.indexes[gvk][field] = extractValue + + return f +} + +// WithStatusSubresource configures the passed object with a status subresource, which means +// calls to Update and Patch will not alter its status. +func (f *ClientBuilder) WithStatusSubresource(o ...client.Object) *ClientBuilder { + f.withStatusSubresource = append(f.withStatusSubresource, o...) + return f +} + +// WithInterceptorFuncs configures the client methods to be intercepted using the provided interceptor.Funcs. +func (f *ClientBuilder) WithInterceptorFuncs(interceptorFuncs interceptor.Funcs) *ClientBuilder { + f.interceptorFuncs = &interceptorFuncs + return f +} + +// WithTypeConverters sets the type converters for the fake client. The list is ordered and the first +// non-erroring converter is used. A type converter must be provided for all types the client is used +// for, otherwise it will error. +// +// This setting is incompatible with WithObjectTracker, as the type converters are a setting on the tracker. +// +// If unset, this defaults to: +// * clientgoapplyconfigurations.NewTypeConverter(scheme.Scheme), +// * managedfields.NewDeducedTypeConverter(), +// +// Be aware that the behavior of the `NewDeducedTypeConverter` might not match the behavior of the +// Kubernetes APIServer, it is recommended to provide a type converter for your types. TypeConverters +// are generated along with ApplyConfigurations. +func (f *ClientBuilder) WithTypeConverters(typeConverters ...managedfields.TypeConverter) *ClientBuilder { + f.typeConverters = append(f.typeConverters, typeConverters...) + return f +} + +// WithReturnManagedFields configures the fake client to return managedFields +// on objects. +func (f *ClientBuilder) WithReturnManagedFields() *ClientBuilder { + f.returnManagedFields = true + return f +} + // Build builds and returns a new fake client. func (f *ClientBuilder) Build() client.WithWatch { + if f.isBuilt { + panic("Build() must not be called multiple times when creating a ClientBuilder") + } if f.scheme == nil { f.scheme = scheme.Scheme } + if f.restMapper == nil { + f.restMapper = meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + } + + withStatusSubResource := sets.New(inTreeResourcesWithStatus()...) + for _, o := range f.withStatusSubresource { + gvk, err := apiutil.GVKForObject(o, f.scheme) + if err != nil { + panic(fmt.Errorf("failed to get gvk for object %T: %w", withStatusSubResource, err)) + } + withStatusSubResource.Insert(gvk) + } + + if f.objectTracker != nil && len(f.typeConverters) > 0 { + panic(errors.New("WithObjectTracker and WithTypeConverters are incompatible")) + } + + var usesFieldManagedObjectTracker bool + if f.objectTracker == nil { + if len(f.typeConverters) == 0 { + // Use corresponding scheme to ensure the converter error + // for types it can't handle. + clientGoScheme := runtime.NewScheme() + if err := scheme.AddToScheme(clientGoScheme); err != nil { + panic(fmt.Sprintf("failed to construct client-go scheme: %v", err)) + } + f.typeConverters = []managedfields.TypeConverter{ + clientgoapplyconfigurations.NewTypeConverter(clientGoScheme), + managedfields.NewDeducedTypeConverter(), + } + } + f.objectTracker = testing.NewFieldManagedObjectTracker( + f.scheme, + serializer.NewCodecFactory(f.scheme).UniversalDecoder(), + multiTypeConverter{upstream: f.typeConverters}, + ) + usesFieldManagedObjectTracker = true + } + tracker := versionedTracker{ + upstream: f.objectTracker, + scheme: f.scheme, + withStatusSubresource: withStatusSubResource, + usesFieldManagedObjectTracker: usesFieldManagedObjectTracker, + } - tracker := versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme} for _, obj := range f.initObject { if err := tracker.Add(obj); err != nil { panic(fmt.Errorf("failed to add object %v to fake client: %w", obj, err)) @@ -139,91 +323,47 @@ func (f *ClientBuilder) Build() client.WithWatch { panic(fmt.Errorf("failed to add runtime object %v to fake client: %w", obj, err)) } } - return &fakeClient{ - tracker: tracker, - scheme: f.scheme, - } -} - -const trackerAddResourceVersion = "999" -func (t versionedTracker) Add(obj runtime.Object) error { - var objects []runtime.Object - if meta.IsListType(obj) { - var err error - objects, err = meta.ExtractList(obj) - if err != nil { - return err - } - } else { - objects = []runtime.Object{obj} + var result client.WithWatch = &fakeClient{ + tracker: tracker, + scheme: f.scheme, + restMapper: f.restMapper, + indexes: f.indexes, + withStatusSubresource: withStatusSubResource, + returnManagedFields: f.returnManagedFields, } - for _, obj := range objects { - accessor, err := meta.Accessor(obj) - if err != nil { - return fmt.Errorf("failed to get accessor for object: %w", err) - } - if accessor.GetResourceVersion() == "" { - // We use a "magic" value of 999 here because this field - // is parsed as uint and and 0 is already used in Update. - // As we can't go lower, go very high instead so this can - // be recognized - accessor.SetResourceVersion(trackerAddResourceVersion) - } - obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) - if err != nil { - return err - } - if err := t.ObjectTracker.Add(obj); err != nil { - return err - } + if f.interceptorFuncs != nil { + result = interceptor.NewClient(result, *f.interceptorFuncs) } - return nil + f.isBuilt = true + return result } -func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { - accessor, err := meta.Accessor(obj) - if err != nil { - return fmt.Errorf("failed to get accessor for object: %v", err) - } - if accessor.GetName() == "" { - return apierrors.NewInvalid( - obj.GetObjectKind().GroupVersionKind().GroupKind(), - accessor.GetName(), - field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) - } - if accessor.GetResourceVersion() != "" { - return apierrors.NewBadRequest("resourceVersion can not be set for Create requests") - } - accessor.SetResourceVersion("1") - obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) - if err != nil { - return err - } - if err := t.ObjectTracker.Create(gvr, obj, ns); err != nil { - accessor.SetResourceVersion("") - return err - } - - return nil -} +const trackerAddResourceVersion = "999" -// convertFromUnstructuredIfNecessary will convert *unstructured.Unstructured for a GVK that is recocnized +// convertFromUnstructuredIfNecessary will convert runtime.Unstructured for a GVK that is recognized // by the schema into the whatever the schema produces with New() for said GVK. -// This is required because the tracker unconditionally saves on manipulations, but it's List() implementation +// This is required because the tracker unconditionally saves on manipulations, but its List() implementation // tries to assign whatever it finds into a ListType it gets from schema.New() - Thus we have to ensure // we save as the very same type, otherwise subsequent List requests will fail. func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (runtime.Object, error) { - u, isUnstructured := o.(*unstructured.Unstructured) - if !isUnstructured || !s.Recognizes(u.GroupVersionKind()) { + u, isUnstructured := o.(runtime.Unstructured) + if !isUnstructured { + return o, nil + } + gvk := o.GetObjectKind().GroupVersionKind() + if !s.Recognizes(gvk) { return o, nil } - typed, err := s.New(u.GroupVersionKind()) + typed, err := s.New(gvk) if err != nil { - return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", u.GroupVersionKind().String(), err) + return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", gvk, err) + } + if _, isTypedUnstructured := typed.(runtime.Unstructured); isTypedUnstructured { + return o, nil } unstructuredSerialized, err := json.Marshal(u) @@ -237,71 +377,18 @@ func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (ru return typed, nil } -func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { - accessor, err := meta.Accessor(obj) - if err != nil { - return fmt.Errorf("failed to get accessor for object: %v", err) - } - - if accessor.GetName() == "" { - return apierrors.NewInvalid( - obj.GetObjectKind().GroupVersionKind().GroupKind(), - accessor.GetName(), - field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) - } - - gvk := obj.GetObjectKind().GroupVersionKind() - if gvk.Empty() { - gvk, err = apiutil.GVKForObject(obj, t.scheme) - if err != nil { - return err - } - } - - oldObject, err := t.ObjectTracker.Get(gvr, ns, accessor.GetName()) - if err != nil { - // If the resource is not found and the resource allows create on update, issue a - // create instead. - if apierrors.IsNotFound(err) && allowsCreateOnUpdate(gvk) { - return t.Create(gvr, obj, ns) - } - return err - } - - oldAccessor, err := meta.Accessor(oldObject) - if err != nil { +func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { return err } - // If the new object does not have the resource version set and it allows unconditional update, - // default it to the resource version of the existing resource - if accessor.GetResourceVersion() == "" && allowsUnconditionalUpdate(gvk) { - accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) - } - if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() { - return apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified")) - } - if oldAccessor.GetResourceVersion() == "" { - oldAccessor.SetResourceVersion("0") - } - intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64) - if err != nil { - return fmt.Errorf("can not convert resourceVersion %q to int: %v", oldAccessor.GetResourceVersion(), err) - } - intResourceVersion++ - accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10)) - if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 { - return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) - } - obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvr, err := getGVRFromObject(obj, c.scheme) if err != nil { return err } - return t.ObjectTracker.Update(gvr, obj, ns) -} - -func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { - gvr, err := getGVRFromObject(obj, c.scheme) + gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err } @@ -310,36 +397,47 @@ func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.O return err } - gvk, err := apiutil.GVKForObject(obj, c.scheme) - if err != nil { - return err - } ta, err := meta.TypeAccessor(o) if err != nil { return err } - ta.SetKind(gvk.Kind) + + // If the final object is unstructuctured, the json + // representation must contain GVK or the apimachinery + // json serializer will error out. ta.SetAPIVersion(gvk.GroupVersion().String()) + ta.SetKind(gvk.Kind) j, err := json.Marshal(o) if err != nil { return err } - decoder := scheme.Codecs.UniversalDecoder() zero(obj) - _, _, err = decoder.Decode(j, nil, obj) - return err + if err := json.Unmarshal(j, obj); err != nil { + return err + } + + if !c.returnManagedFields { + obj.SetManagedFields(nil) + } + + return ensureTypeMeta(obj, gvk) } func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(list); err != nil { + return nil, err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvk, err := apiutil.GVKForObject(list, c.scheme) if err != nil { return nil, err } - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) @@ -349,23 +447,30 @@ func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ... } func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err } - originalKind := gvk.Kind - - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + originalGVK := gvk + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + listGVK := gvk + listGVK.Kind += "List" - if _, isUnstructuredList := obj.(*unstructured.UnstructuredList); isUnstructuredList && !c.scheme.Recognizes(gvk) { + if _, isUnstructuredList := obj.(runtime.Unstructured); isUnstructuredList && !c.scheme.Recognizes(listGVK) { // We need to register the ListKind with UnstructuredList: // https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51 - c.schemeWriteLock.Lock() + c.schemeLock.RUnlock() + c.schemeLock.Lock() c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) - c.schemeWriteLock.Unlock() + c.schemeLock.Unlock() + c.schemeLock.RLock() } listOpts := client.ListOptions{} @@ -377,39 +482,119 @@ func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...cl return err } - ta, err := meta.TypeAccessor(o) + j, err := json.Marshal(o) if err != nil { return err } - ta.SetKind(originalKind) - ta.SetAPIVersion(gvk.GroupVersion().String()) + zero(obj) + if err := ensureTypeMeta(obj, originalGVK); err != nil { + return err + } + objCopy := obj.DeepCopyObject().(client.ObjectList) + if err := json.Unmarshal(j, objCopy); err != nil { + return err + } - j, err := json.Marshal(o) + objs, err := meta.ExtractList(objCopy) if err != nil { return err } - decoder := scheme.Codecs.UniversalDecoder() - zero(obj) - _, _, err = decoder.Decode(j, nil, obj) + + for _, o := range objs { + if err := ensureTypeMeta(o, gvk); err != nil { + return err + } + + if !c.returnManagedFields { + o.(metav1.Object).SetManagedFields(nil) + } + } + + if listOpts.LabelSelector == nil && listOpts.FieldSelector == nil { + return meta.SetList(obj, objs) + } + + // If we're here, either a label or field selector are specified (or both), so before we return + // the list we must filter it. If both selectors are set, they are ANDed. + filteredList, err := c.filterList(objs, gvk, listOpts.LabelSelector, listOpts.FieldSelector) if err != nil { return err } - if listOpts.LabelSelector != nil { - objs, err := meta.ExtractList(obj) + return meta.SetList(obj, filteredList) +} + +func (c *fakeClient) filterList(list []runtime.Object, gvk schema.GroupVersionKind, ls labels.Selector, fs fields.Selector) ([]runtime.Object, error) { + // Filter the objects with the label selector + filteredList := list + if ls != nil { + objsFilteredByLabel, err := objectutil.FilterWithLabels(list, ls) if err != nil { - return err + return nil, err } - filteredObjs, err := objectutil.FilterWithLabels(objs, listOpts.LabelSelector) + filteredList = objsFilteredByLabel + } + + // Filter the result of the previous pass with the field selector + if fs != nil { + objsFilteredByField, err := c.filterWithFields(filteredList, gvk, fs) if err != nil { - return err + return nil, err } - err = meta.SetList(obj, filteredObjs) - if err != nil { - return err + filteredList = objsFilteredByField + } + + return filteredList, nil +} + +func (c *fakeClient) filterWithFields(list []runtime.Object, gvk schema.GroupVersionKind, fs fields.Selector) ([]runtime.Object, error) { + requiresExact := selector.RequiresExactMatch(fs) + if !requiresExact { + return nil, fmt.Errorf(`field selector %s is not in one of the two supported forms "key==val" or "key=val"`, fs) + } + + c.indexesLock.RLock() + defer c.indexesLock.RUnlock() + // Field selection is mimicked via indexes, so there's no sane answer this function can give + // if there are no indexes registered for the GroupVersionKind of the objects in the list. + indexes := c.indexes[gvk] + for _, req := range fs.Requirements() { + if len(indexes) == 0 || indexes[req.Field] == nil { + return nil, fmt.Errorf("List on GroupVersionKind %v specifies selector on field %s, but no "+ + "index with name %s has been registered for GroupVersionKind %v", gvk, req.Field, req.Field, gvk) } } - return nil + + filteredList := make([]runtime.Object, 0, len(list)) + for _, obj := range list { + matches := true + for _, req := range fs.Requirements() { + indexExtractor := indexes[req.Field] + if !c.objMatchesFieldSelector(obj, indexExtractor, req.Value) { + matches = false + break + } + } + if matches { + filteredList = append(filteredList, obj) + } + } + return filteredList, nil +} + +func (c *fakeClient) objMatchesFieldSelector(o runtime.Object, extractIndex client.IndexerFunc, val string) bool { + obj, isClientObject := o.(client.Object) + if !isClientObject { + panic(fmt.Errorf("expected object %v to be of type client.Object, but it's not", o)) + } + + for _, extractedVal := range extractIndex(obj) { + if extractedVal == val { + return true + } + } + + return false } func (c *fakeClient) Scheme() *runtime.Scheme { @@ -417,11 +602,27 @@ func (c *fakeClient) Scheme() *runtime.Scheme { } func (c *fakeClient) RESTMapper() meta.RESTMapper { - // TODO: Implement a fake RESTMapper. - return nil + return c.restMapper +} + +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (c *fakeClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return apiutil.GVKForObject(obj, c.scheme) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (c *fakeClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return apiutil.IsObjectNamespaced(obj, c.scheme, c.restMapper) } func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + createOptions := &client.CreateOptions{} createOptions.ApplyOptions(opts) @@ -447,11 +648,40 @@ func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...clie } accessor.SetName(fmt.Sprintf("%s%s", base, utilrand.String(randomLength))) } + // Ignore attempts to set deletion timestamp + if !accessor.GetDeletionTimestamp().IsZero() { + accessor.SetDeletionTimestamp(nil) + } + + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } - return c.tracker.Create(gvr, obj, accessor.GetNamespace()) + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + + if err := c.tracker.Create(gvr, obj, accessor.GetNamespace(), *createOptions.AsCreateOptions()); err != nil { + // The managed fields tracker sets gvk even on errors + _ = ensureTypeMeta(obj, gvk) + return err + } + + if !c.returnManagedFields { + obj.SetManagedFields(nil) + } + + return ensureTypeMeta(obj, gvk) } func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvr, err := getGVRFromObject(obj, c.scheme) if err != nil { return err @@ -463,6 +693,14 @@ func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie delOptions := client.DeleteOptions{} delOptions.ApplyOptions(opts) + for _, dryRunOpt := range delOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() // Check the ResourceVersion if that Precondition was specified. if delOptions.Preconditions != nil && delOptions.Preconditions.ResourceVersion != nil { name := accessor.GetName() @@ -485,10 +723,17 @@ func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie } } - return c.deleteObject(gvr, accessor) + return c.deleteObjectLocked(gvr, accessor) } func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err @@ -497,6 +742,15 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts .. dcOptions := client.DeleteAllOfOptions{} dcOptions.ApplyOptions(opts) + for _, dryRunOpt := range dcOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + gvr, _ := meta.UnsafeGuessKindToResource(gvk) o, err := c.tracker.List(gvr, gvk, dcOptions.Namespace) if err != nil { @@ -516,7 +770,7 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts .. if err != nil { return err } - err = c.deleteObject(gvr, accessor) + err = c.deleteObjectLocked(gvr, accessor) if err != nil { return err } @@ -525,6 +779,17 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts .. } func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return c.update(obj, false, opts...) +} + +func (c *fakeClient) update(obj client.Object, isStatus bool, opts ...client.UpdateOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + updateOptions := &client.UpdateOptions{} updateOptions.ApplyOptions(opts) @@ -538,71 +803,399 @@ func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...clie if err != nil { return err } + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } accessor, err := meta.Accessor(obj) if err != nil { return err } - return c.tracker.Update(gvr, obj, accessor.GetNamespace()) -} -func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - patchOptions := &client.PatchOptions{} - patchOptions.ApplyOptions(opts) + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() - for _, dryRunOpt := range patchOptions.DryRun { - if dryRunOpt == metav1.DryRunAll { - return nil - } + // Retain managed fields + // We can ignore all errors here since update will fail if we encounter an error. + obj.SetManagedFields(nil) + current, _ := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) + if currentMetaObj, ok := current.(metav1.Object); ok { + obj.SetManagedFields(currentMetaObj.GetManagedFields()) } - gvr, err := getGVRFromObject(obj, c.scheme) - if err != nil { + if err := c.tracker.update(gvr, obj, accessor.GetNamespace(), isStatus, false, *updateOptions.AsUpdateOptions()); err != nil { return err } - accessor, err := meta.Accessor(obj) - if err != nil { - return err + + if !c.returnManagedFields { + obj.SetManagedFields(nil) } - data, err := patch.Data(obj) + + return ensureTypeMeta(obj, gvk) +} + +func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return c.patch(obj, patch, opts...) +} + +func (c *fakeClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + applyOpts := &client.ApplyOptions{} + applyOpts.ApplyOptions(opts) + + data, err := json.Marshal(obj) if err != nil { + return fmt.Errorf("failed to marshal apply configuration: %w", err) + } + + u := &unstructured.Unstructured{} + if err := json.Unmarshal(data, u); err != nil { + return fmt.Errorf("failed to unmarshal apply configuration: %w", err) + } + + applyPatch := &fakeApplyPatch{} + + patchOpts := &client.PatchOptions{} + patchOpts.Raw = applyOpts.AsPatchOptions() + + if err := c.patch(u, applyPatch, patchOpts); err != nil { return err } - reaction := testing.ObjectReaction(c.tracker) - handled, o, err := reaction(testing.NewPatchAction(gvr, accessor.GetNamespace(), accessor.GetName(), patch.Type(), data)) + acJSON, err := json.Marshal(u) if err != nil { + return fmt.Errorf("failed to marshal patched object: %w", err) + } + + // We have to zero the object in case it contained a status and there is a + // status subresource. If its the private `unstructuredApplyConfiguration` + // we can not zero all of it, as that will cause the embedded Unstructured + // to be nil which then causes a NPD in the json.Unmarshal below. + switch reflect.TypeOf(obj).String() { + case "*client.unstructuredApplyConfiguration": + zero(reflect.ValueOf(obj).Elem().FieldByName("Unstructured").Interface()) + default: + zero(obj) + } + if err := json.Unmarshal(acJSON, obj); err != nil { + return fmt.Errorf("failed to unmarshal patched object: %w", err) + } + + return nil +} + +type fakeApplyPatch struct{} + +func (p *fakeApplyPatch) Type() types.PatchType { + return types.ApplyPatchType +} + +func (p *fakeApplyPatch) Data(obj client.Object) ([]byte, error) { + return json.Marshal(obj) +} + +func (c *fakeClient) patch(obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { return err } - if !handled { - panic("tracker could not handle patch method") + + patchOptions := &client.PatchOptions{} + patchOptions.ApplyOptions(opts) + + if errs := validation.ValidatePatchOptions(patchOptions.AsPatchOptions(), patch.Type()); len(errs) > 0 { + return apierrors.NewInvalid(schema.GroupKind{Group: "meta.k8s.io", Kind: "PatchOptions"}, "", errs) + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + + for _, dryRunOpt := range patchOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } } + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + + var isApplyCreate bool + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + oldObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) + if err != nil { + if !apierrors.IsNotFound(err) || patch.Type() != types.ApplyPatchType { + return err + } + oldObj = &unstructured.Unstructured{} + isApplyCreate = true + } + oldAccessor, err := meta.Accessor(oldObj) + if err != nil { + return err + } + + if patch.Type() == types.ApplyPatchType { + if isApplyCreate { + // Overwrite it unconditionally, this matches the apiserver behavior + // which allows to set it on create, but will then ignore it. + obj.SetResourceVersion("1") + } else { + // SSA deletionTimestamp updates are silently ignored + obj.SetDeletionTimestamp(oldAccessor.GetDeletionTimestamp()) + } + } + + data, err := patch.Data(obj) + if err != nil { + return err + } + + action := testing.NewPatchActionWithOptions( + gvr, + accessor.GetNamespace(), + accessor.GetName(), + patch.Type(), + data, + *patchOptions.AsPatchOptions(), + ) + + // Apply is implemented in the tracker and calling it has side-effects + // such as bumping RV and updating managedFields timestamps, hence we + // can not dry-run it. Luckily, the only validation we use it for + // doesn't apply to SSA - Creating objects with non-nil deletionTimestamp + // through SSA is possible and updating the deletionTimestamp is valid, + // but has no effect. + if patch.Type() != types.ApplyPatchType { + // Apply patch without updating object. + // To remain in accordance with the behavior of k8s api behavior, + // a patch must not allow for changes to the deletionTimestamp of an object. + // The reaction() function applies the patch to the object and calls Update(), + // whereas dryPatch() replicates this behavior but skips the call to Update(). + // This ensures that the patch may be rejected if a deletionTimestamp is modified, prior + // to updating the object. + o, err := dryPatch(action, c.tracker) + if err != nil { + return err + } + newObj, err := meta.Accessor(o) + if err != nil { + return err + } + + // Validate that deletionTimestamp has not been changed + if !deletionTimestampEqual(newObj, oldAccessor) { + return fmt.Errorf("rejected patch, metadata.deletionTimestamp immutable") + } + } + + reaction := testing.ObjectReaction(c.tracker) + handled, o, err := reaction(action) + if err != nil { + // The reaction calls tracker.Get after tracker.Apply to return the object, + // but we may have deleted it in tracker.Apply if there was no finalizer + // left. + if apierrors.IsNotFound(err) && + patch.Type() == types.ApplyPatchType && + oldAccessor.GetDeletionTimestamp() != nil && + len(obj.GetFinalizers()) == 0 { + return nil + } + return err + } + if !handled { + panic("tracker could not handle patch method") + } + ta, err := meta.TypeAccessor(o) if err != nil { return err } - ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + ta.SetKind(gvk.Kind) j, err := json.Marshal(o) if err != nil { return err } - decoder := scheme.Codecs.UniversalDecoder() zero(obj) - _, _, err = decoder.Decode(j, nil, obj) - return err + if err := json.Unmarshal(j, obj); err != nil { + return err + } + + if !c.returnManagedFields { + obj.SetManagedFields(nil) + } + + return ensureTypeMeta(obj, gvk) +} + +// Applying a patch results in a deletionTimestamp that is truncated to the nearest second. +// Check that the diff between a new and old deletion timestamp is within a reasonable threshold +// to be considered unchanged. +func deletionTimestampEqual(newObj metav1.Object, obj metav1.Object) bool { + newTime := newObj.GetDeletionTimestamp() + oldTime := obj.GetDeletionTimestamp() + + if newTime == nil || oldTime == nil { + return newTime == oldTime + } + return newTime.Time.Sub(oldTime.Time).Abs() < time.Second +} + +// The behavior of applying the patch is pulled out into dryPatch(), +// which applies the patch and returns an object, but does not Update() the object. +// This function returns a patched runtime object that may then be validated before a call to Update() is executed. +// This results in some code duplication, but was found to be a cleaner alternative than unmarshalling and introspecting the patch data +// and easier than refactoring the k8s client-go method upstream. +// Duplicate of upstream: https://github.com/kubernetes/client-go/blob/783d0d33626e59d55d52bfd7696b775851f92107/testing/fixture.go#L146-L194 +func dryPatch(action testing.PatchActionImpl, tracker testing.ObjectTracker) (runtime.Object, error) { + ns := action.GetNamespace() + gvr := action.GetResource() + + obj, err := tracker.Get(gvr, ns, action.GetName()) + if err != nil { + if apierrors.IsNotFound(err) && action.GetPatchType() == types.ApplyPatchType { + return &unstructured.Unstructured{}, nil + } + return nil, err + } + + old, err := json.Marshal(obj) + if err != nil { + return nil, err + } + + // reset the object in preparation to unmarshal, since unmarshal does not guarantee that fields + // in obj that are removed by patch are cleared + value := reflect.ValueOf(obj) + value.Elem().Set(reflect.New(value.Type().Elem()).Elem()) + + switch action.GetPatchType() { + case types.JSONPatchType: + patch, err := jsonpatch.DecodePatch(action.GetPatch()) + if err != nil { + return nil, err + } + modified, err := patch.Apply(old) + if err != nil { + return nil, err + } + + if err = json.Unmarshal(modified, obj); err != nil { + return nil, err + } + case types.MergePatchType: + modified, err := jsonpatch.MergePatch(old, action.GetPatch()) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(modified, obj); err != nil { + return nil, err + } + case types.StrategicMergePatchType: + mergedByte, err := strategicpatch.StrategicMergePatch(old, action.GetPatch(), obj) + if err != nil { + return nil, err + } + if err = json.Unmarshal(mergedByte, obj); err != nil { + return nil, err + } + case types.ApplyCBORPatchType: + return nil, errors.New("apply CBOR patches are not supported in the fake client") + case types.ApplyPatchType: + return nil, errors.New("bug in controller-runtime: should not end up in dryPatch for SSA") + default: + return nil, fmt.Errorf("%s PatchType is not supported", action.GetPatchType()) + } + return obj, nil +} + +// copyStatusFrom copies the status from old into new +func copyStatusFrom(old, n runtime.Object) error { + oldMapStringAny, err := toMapStringAny(old) + if err != nil { + return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err) + } + newMapStringAny, err := toMapStringAny(n) + if err != nil { + return fmt.Errorf("failed to convert new to *unststructured.Unstructured: %w", err) + } + + newMapStringAny["status"] = oldMapStringAny["status"] + + if err := fromMapStringAny(newMapStringAny, n); err != nil { + return fmt.Errorf("failed to convert back from map[string]any: %w", err) + } + + return nil +} + +// copyFrom copies from old into new +func copyFrom(old, n runtime.Object) error { + oldMapStringAny, err := toMapStringAny(old) + if err != nil { + return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err) + } + if err := fromMapStringAny(oldMapStringAny, n); err != nil { + return fmt.Errorf("failed to convert back from map[string]any: %w", err) + } + + return nil +} + +func toMapStringAny(obj runtime.Object) (map[string]any, error) { + if unstructured, isUnstructured := obj.(*unstructured.Unstructured); isUnstructured { + return unstructured.Object, nil + } + + serialized, err := json.Marshal(obj) + if err != nil { + return nil, err + } + + u := map[string]any{} + return u, json.Unmarshal(serialized, &u) +} + +func fromMapStringAny(u map[string]any, target runtime.Object) error { + if targetUnstructured, isUnstructured := target.(*unstructured.Unstructured); isUnstructured { + targetUnstructured.Object = u + return nil + } + + serialized, err := json.Marshal(u) + if err != nil { + return fmt.Errorf("failed to serialize: %w", err) + } + + zero(target) + if err := json.Unmarshal(serialized, &target); err != nil { + return fmt.Errorf("failed to deserialize: %w", err) + } + + return nil +} + +func (c *fakeClient) Status() client.SubResourceWriter { + return c.SubResource("status") } -func (c *fakeClient) Status() client.StatusWriter { - return &fakeStatusWriter{client: c} +func (c *fakeClient) SubResource(subResource string) client.SubResourceClient { + return &fakeSubResourceClient{client: c, subResource: subResource} } -func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor metav1.Object) error { +func (c *fakeClient) deleteObjectLocked(gvr schema.GroupVersionResource, accessor metav1.Object) error { old, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) if err == nil { oldAccessor, err := meta.Accessor(old) @@ -610,12 +1203,14 @@ func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor meta if len(oldAccessor.GetFinalizers()) > 0 { now := metav1.Now() oldAccessor.SetDeletionTimestamp(&now) - return c.tracker.Update(gvr, old, accessor.GetNamespace()) + // Call update directly with mutability parameter set to true to allow + // changes to deletionTimestamp + return c.tracker.update(gvr, old, accessor.GetNamespace(), false, true, metav1.UpdateOptions{}) } } } - //TODO: implement propagation + // TODO: implement propagation return c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) } @@ -628,20 +1223,152 @@ func getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupV return gvr, nil } -type fakeStatusWriter struct { - client *fakeClient +type fakeSubResourceClient struct { + client *fakeClient + subResource string +} + +func (sw *fakeSubResourceClient) Get(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { + switch sw.subResource { + case subResourceScale: + // Actual client looks up resource, then extracts the scale sub-resource: + // https://github.com/kubernetes/kubernetes/blob/fb6bbc9781d11a87688c398778525c4e1dcb0f08/pkg/registry/apps/deployment/storage/storage.go#L307 + if err := sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { + return err + } + scale, isScale := subResource.(*autoscalingv1.Scale) + if !isScale { + return apierrors.NewBadRequest(fmt.Sprintf("expected Scale, got %T", subResource)) + } + scaleOut, err := extractScale(obj) + if err != nil { + return err + } + *scale = *scaleOut + return nil + default: + return fmt.Errorf("fakeSubResourceClient does not support get for %s", sw.subResource) + } +} + +func (sw *fakeSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + switch sw.subResource { + case "eviction": + _, isEviction := subResource.(*policyv1beta1.Eviction) + if !isEviction { + _, isEviction = subResource.(*policyv1.Eviction) + } + if !isEviction { + return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %T, expected Eviction", subResource)) + } + if _, isPod := obj.(*corev1.Pod); !isPod { + return apierrors.NewNotFound(schema.GroupResource{}, "") + } + + return sw.client.Delete(ctx, obj) + case "token": + tokenRequest, isTokenRequest := subResource.(*authenticationv1.TokenRequest) + if !isTokenRequest { + return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %T, expected TokenRequest", subResource)) + } + if _, isServiceAccount := obj.(*corev1.ServiceAccount); !isServiceAccount { + return apierrors.NewNotFound(schema.GroupResource{}, "") + } + + tokenRequest.Status.Token = "fake-token" + tokenRequest.Status.ExpirationTimestamp = metav1.Date(6041, 1, 1, 0, 0, 0, 0, time.UTC) + + return sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj) + default: + return fmt.Errorf("fakeSubResourceWriter does not support create for %s", sw.subResource) + } +} + +func (sw *fakeSubResourceClient) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + updateOptions := client.SubResourceUpdateOptions{} + updateOptions.ApplyOptions(opts) + + switch sw.subResource { + case subResourceScale: + if err := sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj.DeepCopyObject().(client.Object)); err != nil { + return err + } + if updateOptions.SubResourceBody == nil { + return apierrors.NewBadRequest("missing SubResourceBody") + } + + scale, isScale := updateOptions.SubResourceBody.(*autoscalingv1.Scale) + if !isScale { + return apierrors.NewBadRequest(fmt.Sprintf("expected Scale, got %T", updateOptions.SubResourceBody)) + } + if err := applyScale(obj, scale); err != nil { + return err + } + return sw.client.update(obj, false, &updateOptions.UpdateOptions) + default: + body := obj + if updateOptions.SubResourceBody != nil { + body = updateOptions.SubResourceBody + } + return sw.client.update(body, true, &updateOptions.UpdateOptions) + } } -func (sw *fakeStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { - // TODO(droot): This results in full update of the obj (spec + status). Need - // a way to update status field only. - return sw.client.Update(ctx, obj, opts...) +func (sw *fakeSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + patchOptions := client.SubResourcePatchOptions{} + patchOptions.ApplyOptions(opts) + + body := obj + if patchOptions.SubResourceBody != nil { + body = patchOptions.SubResourceBody + } + + // this is necessary to identify that last call was made for status patch, through stack trace. + if sw.subResource == "status" { + return sw.statusPatch(body, patch, patchOptions) + } + + return sw.client.patch(body, patch, &patchOptions.PatchOptions) } -func (sw *fakeStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - // TODO(droot): This results in full update of the obj (spec + status). Need - // a way to update status field only. - return sw.client.Patch(ctx, obj, patch, opts...) +func (sw *fakeSubResourceClient) statusPatch(body client.Object, patch client.Patch, patchOptions client.SubResourcePatchOptions) error { + return sw.client.patch(body, patch, &patchOptions.PatchOptions) +} + +func (sw *fakeSubResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + if sw.subResource != "status" { + return errors.New("fakeSubResourceClient currently only supports Apply for status subresource") + } + + applyOpts := &client.SubResourceApplyOptions{} + applyOpts.ApplyOpts(opts) + + data, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("failed to marshal apply configuration: %w", err) + } + + u := &unstructured.Unstructured{} + if err := json.Unmarshal(data, u); err != nil { + return fmt.Errorf("failed to unmarshal apply configuration: %w", err) + } + + patchOpts := &client.SubResourcePatchOptions{} + patchOpts.Raw = applyOpts.AsPatchOptions() + + if applyOpts.SubResourceBody != nil { + subResourceBodySerialized, err := json.Marshal(applyOpts.SubResourceBody) + if err != nil { + return fmt.Errorf("failed to serialize subresource body: %w", err) + } + subResourceBody := &unstructured.Unstructured{} + if err := json.Unmarshal(subResourceBodySerialized, subResourceBody); err != nil { + return fmt.Errorf("failed to unmarshal subresource body: %w", err) + } + patchOpts.SubResourceBody = subResourceBody + } + + return sw.Patch(ctx, u, &fakeApplyPatch{}, patchOpts) } func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool { @@ -681,7 +1408,7 @@ func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool { case "PodSecurityPolicy": return true } - case "rbac": + case "rbac.authorization.k8s.io": switch gvk.Kind { case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding": return true @@ -741,6 +1468,44 @@ func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool { return false } +func inTreeResourcesWithStatus() []schema.GroupVersionKind { + return []schema.GroupVersionKind{ + {Version: "v1", Kind: "Namespace"}, + {Version: "v1", Kind: "Node"}, + {Version: "v1", Kind: "PersistentVolumeClaim"}, + {Version: "v1", Kind: "PersistentVolume"}, + {Version: "v1", Kind: "Pod"}, + {Version: "v1", Kind: "ReplicationController"}, + {Version: "v1", Kind: "Service"}, + + {Group: "apps", Version: "v1", Kind: "Deployment"}, + {Group: "apps", Version: "v1", Kind: "DaemonSet"}, + {Group: "apps", Version: "v1", Kind: "ReplicaSet"}, + {Group: "apps", Version: "v1", Kind: "StatefulSet"}, + + {Group: "autoscaling", Version: "v1", Kind: "HorizontalPodAutoscaler"}, + + {Group: "batch", Version: "v1", Kind: "CronJob"}, + {Group: "batch", Version: "v1", Kind: "Job"}, + + {Group: "certificates.k8s.io", Version: "v1", Kind: "CertificateSigningRequest"}, + + {Group: "networking.k8s.io", Version: "v1", Kind: "Ingress"}, + {Group: "networking.k8s.io", Version: "v1", Kind: "NetworkPolicy"}, + + {Group: "policy", Version: "v1", Kind: "PodDisruptionBudget"}, + + {Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"}, + + {Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"}, + + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "FlowSchema"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "PriorityLevelConfiguration"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1", Kind: "FlowSchema"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1", Kind: "PriorityLevelConfiguration"}, + } +} + // zero zeros the value of a pointer. func zero(x interface{}) { if x == nil { @@ -749,3 +1514,216 @@ func zero(x interface{}) { res := reflect.ValueOf(x).Elem() res.Set(reflect.Zero(res.Type())) } + +// getSingleOrZeroOptions returns the single options value in the slice, its +// zero value if the slice is empty, or an error if the slice contains more than +// one option value. +func getSingleOrZeroOptions[T any](opts []T) (opt T, err error) { + switch len(opts) { + case 0: + case 1: + opt = opts[0] + default: + err = fmt.Errorf("expected single or no options value, got %d values", len(opts)) + } + return +} + +func extractScale(obj client.Object) (*autoscalingv1.Scale, error) { + switch obj := obj.(type) { + case *appsv1.Deployment: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + case *appsv1.ReplicaSet: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + case *corev1.ReplicationController: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: labels.Set(obj.Spec.Selector).String(), + }, + }, nil + case *appsv1.StatefulSet: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + default: + // TODO: CRDs https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource + return nil, fmt.Errorf("unimplemented scale subresource for resource %T", obj) + } +} + +func applyScale(obj client.Object, scale *autoscalingv1.Scale) error { + switch obj := obj.(type) { + case *appsv1.Deployment: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *appsv1.ReplicaSet: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *corev1.ReplicationController: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *appsv1.StatefulSet: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + default: + // TODO: CRDs https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource + return fmt.Errorf("unimplemented scale subresource for resource %T", obj) + } + return nil +} + +// AddIndex adds an index to a fake client. It will panic if used with a client that is not a fake client. +// It will error if there is already an index for given object with the same name as field. +// +// It can be used to test code that adds indexes to the cache at runtime. +func AddIndex(c client.Client, obj runtime.Object, field string, extractValue client.IndexerFunc) error { + fakeClient, isFakeClient := c.(*fakeClient) + if !isFakeClient { + panic("AddIndex can only be used with a fake client") + } + fakeClient.indexesLock.Lock() + defer fakeClient.indexesLock.Unlock() + + if fakeClient.indexes == nil { + fakeClient.indexes = make(map[schema.GroupVersionKind]map[string]client.IndexerFunc, 1) + } + + gvk, err := apiutil.GVKForObject(obj, fakeClient.scheme) + if err != nil { + return fmt.Errorf("failed to get gvk for %T: %w", obj, err) + } + + if fakeClient.indexes[gvk] == nil { + fakeClient.indexes[gvk] = make(map[string]client.IndexerFunc, 1) + } + + if fakeClient.indexes[gvk][field] != nil { + return fmt.Errorf("index %s already exists", field) + } + + fakeClient.indexes[gvk][field] = extractValue + + return nil +} + +func (c *fakeClient) addToSchemeIfUnknownAndUnstructuredOrPartial(obj runtime.Object) error { + c.schemeLock.Lock() + defer c.schemeLock.Unlock() + + _, isUnstructured := obj.(*unstructured.Unstructured) + _, isUnstructuredList := obj.(*unstructured.UnstructuredList) + _, isPartial := obj.(*metav1.PartialObjectMetadata) + _, isPartialList := obj.(*metav1.PartialObjectMetadataList) + if !isUnstructured && !isUnstructuredList && !isPartial && !isPartialList { + return nil + } + + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + if !c.scheme.Recognizes(gvk) { + c.scheme.AddKnownTypeWithName(gvk, obj) + } + + return nil +} + +func ensureTypeMeta(obj runtime.Object, gvk schema.GroupVersionKind) error { + ta, err := meta.TypeAccessor(obj) + if err != nil { + return err + } + _, isUnstructured := obj.(runtime.Unstructured) + _, isPartialObject := obj.(*metav1.PartialObjectMetadata) + _, isPartialObjectList := obj.(*metav1.PartialObjectMetadataList) + if !isUnstructured && !isPartialObject && !isPartialObjectList { + ta.SetKind("") + ta.SetAPIVersion("") + return nil + } + + ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + + return nil +} diff --git a/pkg/client/fake/client_suite_test.go b/pkg/client/fake/client_suite_test.go index ac5540106e..66590f0b58 100644 --- a/pkg/client/fake/client_suite_test.go +++ b/pkg/client/fake/client_suite_test.go @@ -19,9 +19,8 @@ package fake import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -29,10 +28,9 @@ import ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Fake client Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Fake client Suite") } var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) -}, 60) +}) diff --git a/pkg/client/fake/client_test.go b/pkg/client/fake/client_test.go index 3d6ea17bed..36722b4ddc 100644 --- a/pkg/client/fake/client_test.go +++ b/pkg/client/fake/client_test.go @@ -20,21 +20,46 @@ import ( "context" "encoding/json" "fmt" + "strconv" + "sync" "time" - . "github.com/onsi/ginkgo" + "github.com/google/go-cmp/cmp" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" + clientgoapplyconfigurations "k8s.io/client-go/applyconfigurations" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + "k8s.io/client-go/kubernetes/fake" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +const ( + machineIDFromStatusUpdate = "machine-id-from-status-update" + cidrFromStatusUpdate = "cidr-from-status-update" ) var _ = Describe("Fake client", func() { @@ -44,22 +69,21 @@ var _ = Describe("Fake client", func() { var cl client.WithWatch BeforeEach(func() { + replicas := int32(1) dep = &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "Deployment", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", Namespace: "ns1", ResourceVersion: trackerAddResourceVersion, }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + }, } dep2 = &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "Deployment", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment-2", Namespace: "ns1", @@ -68,12 +92,11 @@ var _ = Describe("Fake client", func() { }, ResourceVersion: trackerAddResourceVersion, }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + }, } cm = &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-cm", Namespace: "ns2", @@ -85,20 +108,20 @@ var _ = Describe("Fake client", func() { } }) - AssertClientBehavior := func() { - It("should be able to Get", func() { + AssertClientWithoutIndexBehavior := func() { + It("should be able to Get", func(ctx SpecContext) { By("Getting a deployment") namespacedName := types.NamespacedName{ Name: "test-deployment", Namespace: "ns1", } obj := &appsv1.Deployment{} - err := cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err := cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(dep)) }) - It("should be able to Get using unstructured", func() { + It("should be able to Get using unstructured", func(ctx SpecContext) { By("Getting a deployment") namespacedName := types.NamespacedName{ Name: "test-deployment", @@ -107,47 +130,53 @@ var _ = Describe("Fake client", func() { obj := &unstructured.Unstructured{} obj.SetAPIVersion("apps/v1") obj.SetKind("Deployment") - err := cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err := cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) }) - It("should be able to List", func() { + It("should be able to List", func(ctx SpecContext) { By("Listing all deployments in a namespace") list := &appsv1.DeploymentList{} - err := cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err := cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(2)) Expect(list.Items).To(ConsistOf(*dep, *dep2)) }) - It("should be able to List using unstructured list", func() { + It("should be able to List using unstructured list", func(ctx SpecContext) { By("Listing all deployments in a namespace") list := &unstructured.UnstructuredList{} list.SetAPIVersion("apps/v1") list.SetKind("DeploymentList") - err := cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err := cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("apps/v1")) + Expect(list.GetKind()).To(Equal("DeploymentList")) Expect(list.Items).To(HaveLen(2)) }) - It("should be able to List using unstructured list when setting a non-list kind", func() { + It("should be able to List using unstructured list when setting a non-list kind", func(ctx SpecContext) { By("Listing all deployments in a namespace") list := &unstructured.UnstructuredList{} list.SetAPIVersion("apps/v1") list.SetKind("Deployment") - err := cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err := cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("apps/v1")) + Expect(list.GetKind()).To(Equal("Deployment")) Expect(list.Items).To(HaveLen(2)) }) - It("should be able to retrieve registered objects that got manipulated as unstructured", func() { + It("should be able to retrieve registered objects that got manipulated as unstructured", func(ctx SpecContext) { list := func() { By("Listing all endpoints in a namespace") list := &unstructured.UnstructuredList{} list.SetAPIVersion("v1") list.SetKind("EndpointsList") - err := cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err := cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("v1")) + Expect(list.GetKind()).To(Equal("EndpointsList")) Expect(list.Items).To(HaveLen(1)) } @@ -161,192 +190,219 @@ var _ = Describe("Fake client", func() { } By("Adding the object during client initialization") - cl = NewFakeClient(unstructuredEndpoint()) + cl = NewClientBuilder().WithRuntimeObjects(unstructuredEndpoint()).Build() list() - Expect(cl.Delete(context.Background(), unstructuredEndpoint())).To(BeNil()) + Expect(cl.Delete(ctx, unstructuredEndpoint())).To(Succeed()) By("Creating an object") item := unstructuredEndpoint() - err := cl.Create(context.Background(), item) - Expect(err).To(BeNil()) + err := cl.Create(ctx, item) + Expect(err).ToNot(HaveOccurred()) list() By("Updating the object") item.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.Background(), item) - Expect(err).To(BeNil()) + err = cl.Update(ctx, item) + Expect(err).ToNot(HaveOccurred()) list() By("Patching the object") old := item.DeepCopy() item.SetAnnotations(map[string]string{"bar": "baz"}) - err = cl.Patch(context.Background(), item, client.MergeFrom(old)) - Expect(err).To(BeNil()) + err = cl.Patch(ctx, item, client.MergeFrom(old)) + Expect(err).ToNot(HaveOccurred()) list() }) - It("should be able to Create an unregistered type using unstructured", func() { + It("should be able to Create an unregistered type using unstructured", func(ctx SpecContext) { item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v1") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) - Expect(err).To(BeNil()) + err := cl.Create(ctx, item) + Expect(err).ToNot(HaveOccurred()) }) - It("should be able to Get an unregisted type using unstructured", func() { + It("should be able to Get an unregisted type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v2") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) - Expect(err).To(BeNil()) + err := cl.Create(ctx, item) + Expect(err).ToNot(HaveOccurred()) By("Getting and the object") item = &unstructured.Unstructured{} item.SetAPIVersion("custom/v2") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) - Expect(err).To(BeNil()) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) + Expect(err).ToNot(HaveOccurred()) }) - It("should be able to List an unregistered type using unstructured", func() { + It("should be able to List an unregistered type using unstructured with ListKind", func(ctx SpecContext) { list := &unstructured.UnstructuredList{} list.SetAPIVersion("custom/v3") list.SetKind("ImageList") - err := cl.List(context.Background(), list) - Expect(err).To(BeNil()) + err := cl.List(ctx, list) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("custom/v3")) + Expect(list.GetKind()).To(Equal("ImageList")) + Expect(err).ToNot(HaveOccurred()) }) - It("should be able to List an unregistered type using unstructured", func() { + It("should be able to List an unregistered type using unstructured with Kind", func(ctx SpecContext) { list := &unstructured.UnstructuredList{} list.SetAPIVersion("custom/v4") list.SetKind("Image") - err := cl.List(context.Background(), list) - Expect(err).To(BeNil()) + err := cl.List(ctx, list) + Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("custom/v4")) + Expect(list.GetKind()).To(Equal("Image")) }) - It("should be able to Update an unregistered type using unstructured", func() { + It("should be able to Update an unregistered type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v5") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) - Expect(err).To(BeNil()) + err := cl.Create(ctx, item) + Expect(err).ToNot(HaveOccurred()) By("Updating the object") err = unstructured.SetNestedField(item.Object, int64(2), "spec", "replicas") - Expect(err).To(BeNil()) - err = cl.Update(context.Background(), item) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + err = cl.Update(ctx, item) + Expect(err).ToNot(HaveOccurred()) By("Getting the object") item = &unstructured.Unstructured{} item.SetAPIVersion("custom/v5") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) - Expect(err).To(BeNil()) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) + Expect(err).ToNot(HaveOccurred()) By("Inspecting the object") value, found, err := unstructured.NestedInt64(item.Object, "spec", "replicas") - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(found).To(BeTrue()) Expect(value).To(Equal(int64(2))) }) - It("should be able to Patch an unregistered type using unstructured", func() { + It("should be able to Patch an unregistered type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v6") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) - Expect(err).To(BeNil()) + err := cl.Create(ctx, item) + Expect(err).ToNot(HaveOccurred()) By("Updating the object") original := item.DeepCopy() err = unstructured.SetNestedField(item.Object, int64(2), "spec", "replicas") - Expect(err).To(BeNil()) - err = cl.Patch(context.Background(), item, client.MergeFrom(original)) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + err = cl.Patch(ctx, item, client.MergeFrom(original)) + Expect(err).ToNot(HaveOccurred()) By("Getting the object") item = &unstructured.Unstructured{} item.SetAPIVersion("custom/v6") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) - Expect(err).To(BeNil()) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) + Expect(err).ToNot(HaveOccurred()) By("Inspecting the object") value, found, err := unstructured.NestedInt64(item.Object, "spec", "replicas") - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(found).To(BeTrue()) Expect(value).To(Equal(int64(2))) }) - It("should be able to Delete an unregistered type using unstructured", func() { + It("should be able to Delete an unregistered type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v7") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) - Expect(err).To(BeNil()) + err := cl.Create(ctx, item) + Expect(err).ToNot(HaveOccurred()) By("Deleting the object") - err = cl.Delete(context.Background(), item) - Expect(err).To(BeNil()) + err = cl.Delete(ctx, item) + Expect(err).ToNot(HaveOccurred()) By("Getting the object") item = &unstructured.Unstructured{} item.SetAPIVersion("custom/v7") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should support filtering by labels and their values", func() { + It("should be able to retrieve objects by PartialObjectMetadata", func(ctx SpecContext) { + By("Creating a Resource") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + } + err := cl.Create(ctx, secret) + Expect(err).ToNot(HaveOccurred()) + + By("Fetching the resource using a PartialObjectMeta") + partialObjMeta := &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + } + partialObjMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Secret")) + + err = cl.Get(ctx, client.ObjectKeyFromObject(partialObjMeta), partialObjMeta) + Expect(err).ToNot(HaveOccurred()) + + Expect(partialObjMeta.Kind).To(Equal("Secret")) + Expect(partialObjMeta.APIVersion).To(Equal("v1")) + }) + + It("should support filtering by labels and their values", func(ctx SpecContext) { By("Listing deployments with a particular label and value") list := &appsv1.DeploymentList{} - err := cl.List(context.Background(), list, client.InNamespace("ns1"), + err := cl.List(ctx, list, client.InNamespace("ns1"), client.MatchingLabels(map[string]string{ "test-label": "label-value", })) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should support filtering by label existence", func() { + It("should support filtering by label existence", func(ctx SpecContext) { By("Listing deployments with a particular label") list := &appsv1.DeploymentList{} - err := cl.List(context.Background(), list, client.InNamespace("ns1"), + err := cl.List(ctx, list, client.InNamespace("ns1"), client.HasLabels{"test-label"}) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should be able to Create", func() { + It("should be able to Create", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "new-test-cm", Namespace: "ns2", }, } - err := cl.Create(context.Background(), newcm) - Expect(err).To(BeNil()) + err := cl.Create(ctx, newcm) + Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") namespacedName := types.NamespacedName{ @@ -354,13 +410,13 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(newcm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1")) }) - It("should error on create with set resourceVersion", func() { + It("should error on create with set resourceVersion", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -369,58 +425,46 @@ var _ = Describe("Fake client", func() { ResourceVersion: "1", }, } - err := cl.Create(context.Background(), newcm) + err := cl.Create(ctx, newcm) Expect(apierrors.IsBadRequest(err)).To(BeTrue()) }) - It("should not change the submitted object if Create failed", func() { + It("should not change the submitted object if Create failed", func(ctx SpecContext) { By("Trying to create an existing configmap") submitted := cm.DeepCopy() submitted.ResourceVersion = "" submittedReference := submitted.DeepCopy() - err := cl.Create(context.Background(), submitted) - Expect(err).ToNot(BeNil()) + err := cl.Create(ctx, submitted) + Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) - Expect(submitted).To(Equal(submittedReference)) + Expect(submitted).To(BeComparableTo(submittedReference)) }) - It("should error on Create with empty Name", func() { + It("should error on Create with empty Name", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Namespace: "ns2", }, } - err := cl.Create(context.Background(), newcm) + err := cl.Create(ctx, newcm) Expect(err.Error()).To(Equal("ConfigMap \"\" is invalid: metadata.name: Required value: name is required")) }) - It("should error on Update with empty Name", func() { + It("should error on Update with empty Name", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Namespace: "ns2", }, } - err := cl.Update(context.Background(), newcm) + err := cl.Update(ctx, newcm) Expect(err.Error()).To(Equal("ConfigMap \"\" is invalid: metadata.name: Required value: name is required")) }) - It("should be able to Create with GenerateName", func() { + It("should be able to Create with GenerateName", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ GenerateName: "new-test-cm", Namespace: "ns2", @@ -429,27 +473,23 @@ var _ = Describe("Fake client", func() { }, }, } - err := cl.Create(context.Background(), newcm) - Expect(err).To(BeNil()) + err := cl.Create(ctx, newcm) + Expect(err).ToNot(HaveOccurred()) By("Listing configmaps with a particular label") list := &corev1.ConfigMapList{} - err = cl.List(context.Background(), list, client.InNamespace("ns2"), + err = cl.List(ctx, list, client.InNamespace("ns2"), client.MatchingLabels(map[string]string{ "test-label": "label-value", })) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items[0].Name).NotTo(BeEmpty()) }) - It("should be able to Update", func() { + It("should be able to Update", func(ctx SpecContext) { By("Updating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-cm", Namespace: "ns2", @@ -459,8 +499,8 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm) - Expect(err).To(BeNil()) + err := cl.Update(ctx, newcm) + Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") namespacedName := types.NamespacedName{ @@ -468,19 +508,15 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(newcm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1000")) }) - It("should allow updates with non-set ResourceVersion for a resource that allows unconditional updates", func() { + It("should allow updates with non-set ResourceVersion for a resource that allows unconditional updates", func(ctx SpecContext) { By("Updating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-cm", Namespace: "ns2", @@ -489,8 +525,8 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm) - Expect(err).To(BeNil()) + err := cl.Update(ctx, newcm) + Expect(err).ToNot(HaveOccurred()) By("Getting the configmap") namespacedName := types.NamespacedName{ @@ -498,19 +534,42 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(newcm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1000")) }) - It("should reject updates with non-set ResourceVersion for a resource that doesn't allow unconditional updates", func() { + It("should allow patch when the patch sets RV to 'null'", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + original := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "obj", + Namespace: "ns2", + }} + + err := cl.Create(ctx, original) + Expect(err).ToNot(HaveOccurred()) + + newObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: original.Name, + Namespace: original.Namespace, + Annotations: map[string]string{ + "foo": "bar", + }, + }} + + Expect(cl.Patch(ctx, newObj, client.MergeFrom(original))).To(Succeed()) + + patched := &corev1.ConfigMap{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(original), patched)).To(Succeed()) + Expect(patched.Annotations).To(Equal(map[string]string{"foo": "bar"})) + }) + + It("should reject updates with non-set ResourceVersion for a resource that doesn't allow unconditional updates", func(ctx SpecContext) { By("Creating a new binding") binding := &corev1.Binding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Binding", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-binding", Namespace: "ns2", @@ -522,14 +581,10 @@ var _ = Describe("Fake client", func() { Name: cm.Name, }, } - Expect(cl.Create(context.Background(), binding)).To(Succeed()) + Expect(cl.Create(ctx, binding)).To(Succeed()) By("Updating the binding with a new resource lacking resource version") newBinding := &corev1.Binding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Binding", - }, ObjectMeta: metav1.ObjectMeta{ Name: binding.Name, Namespace: binding.Namespace, @@ -539,23 +594,19 @@ var _ = Describe("Fake client", func() { Name: "blue", }, } - Expect(cl.Update(context.Background(), newBinding)).NotTo(Succeed()) + Expect(cl.Update(ctx, newBinding)).NotTo(Succeed()) }) - It("should allow create on update for a resource that allows create on update", func() { + It("should allow create on update for a resource that allows create on update", func(ctx SpecContext) { By("Creating a new lease with update") lease := &coordinationv1.Lease{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "coordination.k8s.io/v1", - Kind: "Lease", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-lease", Namespace: "ns2", }, Spec: coordinationv1.LeaseSpec{}, } - Expect(cl.Create(context.Background(), lease)).To(Succeed()) + Expect(cl.Create(ctx, lease)).To(Succeed()) By("Getting the lease") namespacedName := types.NamespacedName{ @@ -563,18 +614,14 @@ var _ = Describe("Fake client", func() { Namespace: lease.Namespace, } obj := &coordinationv1.Lease{} - Expect(cl.Get(context.Background(), namespacedName, obj)).To(Succeed()) + Expect(cl.Get(ctx, namespacedName, obj)).To(Succeed()) Expect(obj).To(Equal(lease)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1")) }) - It("should reject create on update for a resource that does not allow create on update", func() { + It("should reject create on update for a resource that does not allow create on update", func(ctx SpecContext) { By("Attemping to create a new configmap with update") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "different-test-cm", Namespace: "ns2", @@ -583,10 +630,10 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - Expect(cl.Update(context.Background(), newcm)).NotTo(Succeed()) + Expect(cl.Update(ctx, newcm)).NotTo(Succeed()) }) - It("should reject updates with non-matching ResourceVersion", func() { + It("should reject updates with non-matching ResourceVersion", func(ctx SpecContext) { By("Updating a new configmap") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -598,7 +645,7 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm) + err := cl.Update(ctx, newcm) Expect(apierrors.IsConflict(err)).To(BeTrue()) By("Getting the configmap") @@ -607,67 +654,109 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj).To(Equal(cm)) + Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) + }) + + It("should reject apply with non-matching ResourceVersion", func(ctx SpecContext) { + cl := NewClientBuilder().WithRuntimeObjects(cm).Build() + applyCM := corev1applyconfigurations.ConfigMap(cm.Name, cm.Namespace).WithResourceVersion("0") + err := cl.Apply(ctx, applyCM, client.FieldOwner("test")) + Expect(apierrors.IsConflict(err)).To(BeTrue()) + + obj := &corev1.ConfigMap{} + err = cl.Get(ctx, client.ObjectKeyFromObject(cm), obj) + Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(cm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) }) - It("should reject Delete with a mismatched ResourceVersion", func() { + It("should reject Delete with a mismatched ResourceVersion", func(ctx SpecContext) { bogusRV := "bogus" By("Deleting with a mismatched ResourceVersion Precondition") - err := cl.Delete(context.Background(), dep, client.Preconditions{ResourceVersion: &bogusRV}) + err := cl.Delete(ctx, dep, client.Preconditions{ResourceVersion: &bogusRV}) Expect(apierrors.IsConflict(err)).To(BeTrue()) list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err = cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(2)) Expect(list.Items).To(ConsistOf(*dep, *dep2)) }) - It("should successfully Delete with a matching ResourceVersion", func() { + It("should successfully Delete with a matching ResourceVersion", func(ctx SpecContext) { goodRV := trackerAddResourceVersion By("Deleting with a matching ResourceVersion Precondition") - err := cl.Delete(context.Background(), dep, client.Preconditions{ResourceVersion: &goodRV}) - Expect(err).To(BeNil()) + err := cl.Delete(ctx, dep, client.Preconditions{ResourceVersion: &goodRV}) + Expect(err).ToNot(HaveOccurred()) list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err = cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should be able to Delete with no ResourceVersion Precondition", func() { + It("should be able to Delete with no ResourceVersion Precondition", func(ctx SpecContext) { By("Deleting a deployment") - err := cl.Delete(context.Background(), dep) - Expect(err).To(BeNil()) + err := cl.Delete(ctx, dep) + Expect(err).ToNot(HaveOccurred()) By("Listing all deployments in the namespace") list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err = cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should be able to Delete with no opts even if object's ResourceVersion doesn't match server", func() { + It("should be able to Delete with no opts even if object's ResourceVersion doesn't match server", func(ctx SpecContext) { By("Deleting a deployment") depCopy := dep.DeepCopy() depCopy.ResourceVersion = "bogus" - err := cl.Delete(context.Background(), depCopy) - Expect(err).To(BeNil()) + err := cl.Delete(ctx, depCopy) + Expect(err).ToNot(HaveOccurred()) By("Listing all deployments in the namespace") list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err = cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should handle finalizers on Update", func() { + It("should handle finalizers in Apply ", func(ctx SpecContext) { + cl := client.WithFieldOwner(cl, "test") + + By("Creating the object with a finalizer") + cm := corev1applyconfigurations.ConfigMap("test-cm", "delete-with-finalizers"). + WithFinalizers("finalizers.sigs.k8s.io/test") + Expect(cl.Apply(ctx, cm)).To(Succeed()) + + By("Deleting the object") + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: *cm.Name, + Namespace: *cm.Namespace, + }} + Expect(cl.Delete(ctx, obj)).NotTo(HaveOccurred()) + + By("Getting the object") + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).NotTo(HaveOccurred()) + Expect(obj.DeletionTimestamp).NotTo(BeNil()) + + By("Removing the finalizer through SSA") + cm.ResourceVersion = nil + cm.Finalizers = nil + Expect(cl.Apply(ctx, cm)).NotTo(HaveOccurred()) + + By("Getting the object") + err := cl.Get(ctx, client.ObjectKeyFromObject(obj), &corev1.ConfigMap{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should handle finalizers on Update", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "delete-with-finalizers", @@ -683,43 +772,91 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) - Expect(err).To(BeNil()) + err := cl.Create(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) By("Deleting the object") - err = cl.Delete(context.Background(), newObj) - Expect(err).To(BeNil()) + err = cl.Delete(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) Expect(obj.DeletionTimestamp).NotTo(BeNil()) By("Removing the finalizer") obj.Finalizers = []string{} - err = cl.Update(context.Background(), obj) - Expect(err).To(BeNil()) + err = cl.Update(ctx, obj) + Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj = &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should be able to Delete a Collection", func() { + It("should reject changes to deletionTimestamp on Update", func(ctx SpecContext) { + namespacedName := types.NamespacedName{ + Name: "test-cm", + Namespace: "reject-with-deletiontimestamp", + } + By("Updating a new object") + newObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + }, + Data: map[string]string{ + "test-key": "new-value", + }, + } + err := cl.Create(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) + + By("Getting the object") + obj := &corev1.ConfigMap{} + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj.DeletionTimestamp).To(BeNil()) + + By("Adding deletionTimestamp") + now := metav1.Now() + obj.DeletionTimestamp = &now + err = cl.Update(ctx, obj) + Expect(err).To(HaveOccurred()) + + By("Deleting the object") + err = cl.Delete(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) + + By("Changing the deletionTimestamp to new value") + obj = &corev1.ConfigMap{} + t := metav1.NewTime(time.Now().Add(time.Second)) + obj.DeletionTimestamp = &t + err = cl.Update(ctx, obj) + Expect(err).To(HaveOccurred()) + + By("Removing deletionTimestamp") + obj.DeletionTimestamp = nil + err = cl.Update(ctx, obj) + Expect(err).To(HaveOccurred()) + + }) + + It("should be able to Delete a Collection", func(ctx SpecContext) { By("Deleting a deploymentList") - err := cl.DeleteAllOf(context.Background(), &appsv1.Deployment{}, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err := cl.DeleteAllOf(ctx, &appsv1.Deployment{}, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) By("Listing all deployments in the namespace") list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) - Expect(err).To(BeNil()) + err = cl.List(ctx, list, client.InNamespace("ns1")) + Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(BeEmpty()) }) - It("should handle finalizers deleting a collection", func() { + It("should handle finalizers deleting a collection", func(ctx SpecContext) { for i := 0; i < 5; i++ { namespacedName := types.NamespacedName{ Name: fmt.Sprintf("test-cm-%d", i), @@ -736,27 +873,27 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) - Expect(err).To(BeNil()) + err := cl.Create(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) } By("Deleting the object") - err := cl.DeleteAllOf(context.Background(), &corev1.ConfigMap{}, client.InNamespace("delete-collection-with-finalizers")) - Expect(err).To(BeNil()) + err := cl.DeleteAllOf(ctx, &corev1.ConfigMap{}, client.InNamespace("delete-collection-with-finalizers")) + Expect(err).ToNot(HaveOccurred()) configmaps := corev1.ConfigMapList{} - err = cl.List(context.Background(), &configmaps, client.InNamespace("delete-collection-with-finalizers")) - Expect(err).To(BeNil()) + err = cl.List(ctx, &configmaps, client.InNamespace("delete-collection-with-finalizers")) + Expect(err).ToNot(HaveOccurred()) - Expect(len(configmaps.Items)).To(Equal(5)) + Expect(configmaps.Items).To(HaveLen(5)) for _, cm := range configmaps.Items { Expect(cm.DeletionTimestamp).NotTo(BeNil()) } }) - It("should be able to watch", func() { + It("should be able to watch", func(ctx SpecContext) { By("Creating a watch") - objWatch, err := cl.Watch(context.Background(), &corev1.ServiceList{}) + objWatch, err := cl.Watch(ctx, &corev1.ServiceList{}) Expect(err).NotTo(HaveOccurred()) defer objWatch.Stop() @@ -767,7 +904,7 @@ var _ = Describe("Fake client", func() { // in the outer routine, sleep to make sure this is always true time.Sleep(100 * time.Millisecond) - err := cl.Create(context.Background(), &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "for-watch"}}) + err := cl.Create(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "for-watch"}}) Expect(err).ToNot(HaveOccurred()) }() @@ -781,7 +918,7 @@ var _ = Describe("Fake client", func() { }) Context("with the DryRun option", func() { - It("should not create a new object", func() { + It("should not create a new object", func(ctx SpecContext) { By("Creating a new configmap with DryRun") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -789,8 +926,8 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", }, } - err := cl.Create(context.Background(), newcm, client.DryRunAll) - Expect(err).To(BeNil()) + err := cl.Create(ctx, newcm, client.DryRunAll) + Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") namespacedName := types.NamespacedName{ @@ -798,13 +935,13 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) Expect(obj).NotTo(Equal(newcm)) }) - It("should not Update the object", func() { + It("should not Update the object", func(ctx SpecContext) { By("Updating a new configmap with DryRun") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -816,8 +953,8 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm, client.DryRunAll) - Expect(err).To(BeNil()) + err := cl.Update(ctx, newcm, client.DryRunAll) + Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") namespacedName := types.NamespacedName{ @@ -825,14 +962,35 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) - Expect(err).To(BeNil()) + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj).To(Equal(cm)) + Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) + }) + + It("Should not Delete the object", func(ctx SpecContext) { + By("Deleting a configmap with DryRun with Delete()") + err := cl.Delete(ctx, cm, client.DryRunAll) + Expect(err).ToNot(HaveOccurred()) + + By("Deleting a configmap with DryRun with DeleteAllOf()") + err = cl.DeleteAllOf(ctx, cm, client.DryRunAll) + Expect(err).ToNot(HaveOccurred()) + + By("Getting the configmap") + namespacedName := types.NamespacedName{ + Name: "test-cm", + Namespace: "ns2", + } + obj := &corev1.ConfigMap{} + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(cm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) }) }) - It("should be able to Patch", func() { + It("should be able to Patch", func(ctx SpecContext) { By("Patching a deployment") mergePatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ @@ -842,7 +1000,7 @@ var _ = Describe("Fake client", func() { }, }) Expect(err).NotTo(HaveOccurred()) - err = cl.Patch(context.Background(), dep, client.RawPatch(types.StrategicMergePatchType, mergePatch)) + err = cl.Patch(ctx, dep, client.RawPatch(types.StrategicMergePatchType, mergePatch)) Expect(err).NotTo(HaveOccurred()) By("Getting the patched deployment") @@ -851,18 +1009,18 @@ var _ = Describe("Fake client", func() { Namespace: "ns1", } obj := &appsv1.Deployment{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).NotTo(HaveOccurred()) Expect(obj.Annotations["foo"]).To(Equal("bar")) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1000")) }) - It("should handle finalizers on Patch", func() { + It("should ignore deletionTimestamp without finalizer on Create", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", - Namespace: "delete-with-finalizers", + Namespace: "ignore-deletiontimestamp", } - By("Updating a new object") + By("Creating a new object") now := metav1.Now() newObj := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -875,10 +1033,81 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) - Expect(err).To(BeNil()) - By("Removing the finalizer") + err := cl.Create(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) + + By("Getting the object") + obj := &corev1.ConfigMap{} + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj.DeletionTimestamp).To(BeNil()) + + }) + + It("should reject deletionTimestamp without finalizers on Build", func(ctx SpecContext) { + namespacedName := types.NamespacedName{ + Name: "test-cm", + Namespace: "reject-deletiontimestamp-no-finalizers", + } + By("Build with a new object without finalizer") + now := metav1.Now() + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + DeletionTimestamp: &now, + }, + Data: map[string]string{ + "test-key": "new-value", + }, + } + + Expect(func() { NewClientBuilder().WithObjects(obj).Build() }).To(Panic()) + + By("Build with a new object with finalizer") + newObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + Finalizers: []string{"finalizers.sigs.k8s.io/test"}, + DeletionTimestamp: &now, + }, + Data: map[string]string{ + "test-key": "new-value", + }, + } + + cl := NewClientBuilder().WithObjects(newObj).Build() + + By("Getting the object") + obj = &corev1.ConfigMap{} + err := cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + + }) + + It("should reject changes to deletionTimestamp on Patch", func(ctx SpecContext) { + namespacedName := types.NamespacedName{ + Name: "test-cm", + Namespace: "reject-deletiontimestamp", + } + By("Creating a new object") + now := metav1.Now() + newObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + Finalizers: []string{"finalizers.sigs.k8s.io/test"}, + }, + Data: map[string]string{ + "test-key": "new-value", + }, + } + err := cl.Create(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) + + By("Add a deletionTimestamp") obj := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: namespacedName.Name, @@ -887,17 +1116,80 @@ var _ = Describe("Fake client", func() { DeletionTimestamp: &now, }, } - obj.Finalizers = []string{} - err = cl.Patch(context.Background(), obj, client.MergeFrom(newObj)) - Expect(err).To(BeNil()) + err = cl.Patch(ctx, obj, client.MergeFrom(newObj)) + Expect(err).To(HaveOccurred()) + + By("Deleting the object") + err = cl.Delete(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) + + By("Getting the object") + obj = &corev1.ConfigMap{} + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj.DeletionTimestamp).NotTo(BeNil()) + + By("Changing the deletionTimestamp to new value") + newObj = &corev1.ConfigMap{} + t := metav1.NewTime(time.Now().Add(time.Second)) + newObj.DeletionTimestamp = &t + err = cl.Patch(ctx, newObj, client.MergeFrom(obj)) + Expect(err).To(HaveOccurred()) + + By("Removing deletionTimestamp") + newObj = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + DeletionTimestamp: nil, + }, + } + err = cl.Patch(ctx, newObj, client.MergeFrom(obj)) + Expect(err).To(HaveOccurred()) + + }) + + It("should handle finalizers on Patch", func(ctx SpecContext) { + namespacedName := types.NamespacedName{ + Name: "test-cm", + Namespace: "delete-with-finalizers", + } + By("Creating a new object") + newObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + Finalizers: []string{"finalizers.sigs.k8s.io/test"}, + }, + Data: map[string]string{ + "test-key": "new-value", + }, + } + err := cl.Create(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) + + By("Deleting the object") + err = cl.Delete(ctx, newObj) + Expect(err).ToNot(HaveOccurred()) + + By("Removing the finalizer") + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + Finalizers: []string{}, + }, + } + err = cl.Patch(ctx, obj, client.MergeFrom(newObj)) + Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj = &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should remove finalizers of the object on Patch", func() { + It("should remove finalizers of the object on Patch", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "patch-finalizers-in-obj", @@ -913,8 +1205,8 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), obj) - Expect(err).To(BeNil()) + err := cl.Create(ctx, obj) + Expect(err).ToNot(HaveOccurred()) By("Removing the finalizer") mergePatch, err := json.Marshal(map[string]interface{}{ @@ -924,19 +1216,20 @@ var _ = Describe("Fake client", func() { }, }, }) - Expect(err).To(BeNil()) - err = cl.Patch(context.Background(), obj, client.RawPatch(types.StrategicMergePatchType, mergePatch)) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + err = cl.Patch(ctx, obj, client.RawPatch(types.StrategicMergePatchType, mergePatch)) + Expect(err).ToNot(HaveOccurred()) By("Check the finalizer has been removed in the object") - Expect(len(obj.Finalizers)).To(Equal(0)) + Expect(obj.Finalizers).To(BeEmpty()) By("Check the finalizer has been removed in client") newObj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, newObj) - Expect(err).To(BeNil()) - Expect(len(newObj.Finalizers)).To(Equal(0)) + err = cl.Get(ctx, namespacedName, newObj) + Expect(err).ToNot(HaveOccurred()) + Expect(newObj.Finalizers).To(BeEmpty()) }) + } Context("with default scheme.Scheme", func() { @@ -945,7 +1238,7 @@ var _ = Describe("Fake client", func() { WithObjects(dep, dep2, cm). Build() }) - AssertClientBehavior() + AssertClientWithoutIndexBehavior() }) Context("with given scheme", func() { @@ -960,25 +1253,2068 @@ var _ = Describe("Fake client", func() { WithLists(&appsv1.DeploymentList{Items: []appsv1.Deployment{*dep, *dep2}}). Build() }) - AssertClientBehavior() + AssertClientWithoutIndexBehavior() }) - It("should set the ResourceVersion to 999 when adding an object to the tracker", func() { - cl := NewClientBuilder().WithObjects(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}).Build() - - retrieved := &corev1.Secret{} - Expect(cl.Get(context.Background(), types.NamespacedName{Name: "cm"}, retrieved)).To(Succeed()) + Context("with Indexes", func() { + depReplicasIndexer := func(obj client.Object) []string { + dep, ok := obj.(*appsv1.Deployment) + if !ok { + panic(fmt.Errorf("indexer function for type %T's spec.replicas field received"+ + " object of type %T, this should never happen", appsv1.Deployment{}, obj)) + } + indexVal := "" + if dep.Spec.Replicas != nil { + indexVal = strconv.Itoa(int(*dep.Spec.Replicas)) + } + return []string{indexVal} + } - reference := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "cm", - ResourceVersion: "999", - }, + depStrategyTypeIndexer := func(obj client.Object) []string { + dep, ok := obj.(*appsv1.Deployment) + if !ok { + panic(fmt.Errorf("indexer function for type %T's spec.strategy.type field received"+ + " object of type %T, this should never happen", appsv1.Deployment{}, obj)) + } + return []string{string(dep.Spec.Strategy.Type)} + } + + var cb *ClientBuilder + BeforeEach(func() { + cb = NewClientBuilder(). + WithObjects(dep, dep2, cm). + WithIndex(&appsv1.Deployment{}, "spec.replicas", depReplicasIndexer) + }) + + Context("client has just one Index", func() { + BeforeEach(func() { cl = cb.Build() }) + + Context("behavior that doesn't use an Index", func() { + AssertClientWithoutIndexBehavior() + }) + + Context("filtered List using field selector", func() { + It("errors when there's no Index for the GroupVersionResource", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("key", "val"), + } + list := &corev1.ConfigMapList{} + err := cl.List(ctx, list, listOpts) + Expect(err).To(HaveOccurred()) + Expect(list.Items).To(BeEmpty()) + }) + + It("errors when there's no Index for the GroupVersionResource with UnstructuredList", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("key", "val"), + } + list := &unstructured.UnstructuredList{} + list.SetAPIVersion("v1") + list.SetKind("ConfigMapList") + err := cl.List(ctx, list, listOpts) + Expect(err).To(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("v1")) + Expect(list.GetKind()).To(Equal("ConfigMapList")) + Expect(list.Items).To(BeEmpty()) + }) + + It("errors when there's no Index matching the field name", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.paused", "false"), + } + list := &appsv1.DeploymentList{} + err := cl.List(ctx, list, listOpts) + Expect(err).To(HaveOccurred()) + Expect(list.Items).To(BeEmpty()) + }) + + It("errors when field selector uses two requirements", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermEqualSelector("spec.replicas", "1"), + fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), + )} + list := &appsv1.DeploymentList{} + err := cl.List(ctx, list, listOpts) + Expect(err).To(HaveOccurred()) + Expect(list.Items).To(BeEmpty()) + }) + + It("returns two deployments that match the only field selector requirement", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep, *dep2)) + }) + + It("returns no object because no object matches the only field selector requirement", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + + It("returns deployment that matches both the field and label selectors", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), + LabelSelector: labels.SelectorFromSet(dep2.Labels), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep2)) + }) + + It("returns no object even if field selector matches because label selector doesn't", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), + LabelSelector: labels.Nothing(), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + + It("returns no object even if label selector matches because field selector doesn't", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), + LabelSelector: labels.Everything(), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + + It("supports adding an index at runtime", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", "test-deployment-2"), + } + list := &appsv1.DeploymentList{} + err := cl.List(ctx, list, listOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no index with name metadata.name has been registered")) + + err = AddIndex(cl, &appsv1.Deployment{}, "metadata.name", func(obj client.Object) []string { + return []string{obj.GetName()} + }) + Expect(err).To(Succeed()) + + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep2)) + }) + + It("Is not a datarace to add and use indexes in parallel", func(ctx SpecContext) { + wg := sync.WaitGroup{} + wg.Add(2) + + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), + } + go func() { + defer wg.Done() + defer GinkgoRecover() + Expect(cl.List(ctx, &appsv1.DeploymentList{}, listOpts)).To(Succeed()) + }() + go func() { + defer wg.Done() + defer GinkgoRecover() + err := AddIndex(cl, &appsv1.Deployment{}, "metadata.name", func(obj client.Object) []string { + return []string{obj.GetName()} + }) + Expect(err).To(Succeed()) + }() + wg.Wait() + }) + }) + }) + + Context("client has two Indexes", func() { + BeforeEach(func() { + cl = cb.WithIndex(&appsv1.Deployment{}, "spec.strategy.type", depStrategyTypeIndexer).Build() + }) + + Context("behavior that doesn't use an Index", func() { + AssertClientWithoutIndexBehavior() + }) + + Context("filtered List using field selector", func() { + It("uses the second index to retrieve the indexed objects when there are matches", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep)) + }) + + It("uses the second index to retrieve the indexed objects when there are no matches", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RollingUpdateDeploymentStrategyType)), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + + It("no error when field selector uses two requirements", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermEqualSelector("spec.replicas", "1"), + fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), + )} + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep)) + }) + }) + }) + }) + + It("should set the ResourceVersion to 999 when adding an object to the tracker", func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}).Build() + + retrieved := &corev1.Secret{} + Expect(cl.Get(ctx, types.NamespacedName{Name: "cm"}, retrieved)).To(Succeed()) + + reference := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cm", + ResourceVersion: "999", + }, } Expect(retrieved).To(Equal(reference)) }) + + It("should be able to build with given tracker and get resource", func(ctx SpecContext) { + clientSet := fake.NewSimpleClientset(dep) + cl := NewClientBuilder().WithRuntimeObjects(dep2).WithObjectTracker(clientSet.Tracker()).Build() + + By("Getting a deployment") + namespacedName := types.NamespacedName{ + Name: "test-deployment", + Namespace: "ns1", + } + obj := &appsv1.Deployment{} + err := cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj).To(BeComparableTo(dep)) + + By("Getting a deployment from clientSet") + csDep2, err := clientSet.AppsV1().Deployments("ns1").Get(ctx, "test-deployment-2", metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(csDep2).To(Equal(dep2)) + + By("Getting a new deployment") + namespacedName3 := types.NamespacedName{ + Name: "test-deployment-3", + Namespace: "ns1", + } + + dep3 := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-3", + Namespace: "ns1", + Labels: map[string]string{ + "test-label": "label-value", + }, + ResourceVersion: trackerAddResourceVersion, + }, + } + + _, err = clientSet.AppsV1().Deployments("ns1").Create(ctx, dep3, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + + obj = &appsv1.Deployment{} + err = cl.Get(ctx, namespacedName3, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj).To(BeComparableTo(dep3)) + }) + + It("should not change the status of typed objects that have a status subresource on update", func(ctx SpecContext) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + obj.Status.Phase = "Running" + Expect(cl.Update(ctx, obj)).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Status).To(BeEquivalentTo(corev1.PodStatus{})) + }) + + It("should return a conflict error when an incorrect RV is used on status update", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + ResourceVersion: trackerAddResourceVersion, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + obj.Status.Phase = corev1.NodeRunning + obj.ResourceVersion = "invalid" + err := cl.Status().Update(ctx, obj) + Expect(apierrors.IsConflict(err)).To(BeTrue()) + }) + + It("should not change non-status field of typed objects that have a status subresource on status update", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + MachineID: "machine-id", + }, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + objOriginal := obj.DeepCopy() + + obj.Spec.PodCIDR = cidrFromStatusUpdate + obj.Annotations = map[string]string{ + "some-annotation-key": "some-annotation-value", + } + obj.Labels = map[string]string{ + "some-label-key": "some-label-value", + } + + obj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + + objOriginal.APIVersion = actual.APIVersion + objOriginal.Kind = actual.Kind + objOriginal.ResourceVersion = actual.ResourceVersion + objOriginal.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cmp.Diff(objOriginal, actual)).To(BeEmpty()) + }) + + It("should be able to update an object after updating an object's status", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + MachineID: "machine-id", + }, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + expectedObj := obj.DeepCopy() + + obj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) + + obj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + expectedObj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + Expect(cl.Update(ctx, obj)).NotTo(HaveOccurred()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + + expectedObj.APIVersion = actual.APIVersion + expectedObj.Kind = actual.Kind + expectedObj.ResourceVersion = actual.ResourceVersion + expectedObj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cmp.Diff(expectedObj, actual)).To(BeEmpty()) + }) + + It("should be able to update an object's status after updating an object", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + MachineID: "machine-id", + }, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + expectedObj := obj.DeepCopy() + + obj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + expectedObj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + Expect(cl.Update(ctx, obj)).NotTo(HaveOccurred()) + + obj.Spec.PodCIDR = cidrFromStatusUpdate + obj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + + expectedObj.APIVersion = actual.APIVersion + expectedObj.Kind = actual.Kind + expectedObj.ResourceVersion = actual.ResourceVersion + expectedObj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cmp.Diff(expectedObj, actual)).To(BeEmpty()) + }) + + It("Should only override status fields of typed objects that have a status subresource on status update", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + MachineID: "machine-id", + }, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + objOriginal := obj.DeepCopy() + + obj.Status.Phase = corev1.NodeRunning + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + + objOriginal.APIVersion = actual.APIVersion + objOriginal.Kind = actual.Kind + objOriginal.ResourceVersion = actual.ResourceVersion + Expect(cmp.Diff(objOriginal, actual)).ToNot(BeEmpty()) + Expect(objOriginal.Status.NodeInfo.MachineID).To(Equal(actual.Status.NodeInfo.MachineID)) + Expect(objOriginal.Status.Phase).ToNot(Equal(actual.Status.Phase)) + }) + + It("should be able to change typed objects that have a scale subresource on patch", func(ctx SpecContext) { + obj := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deploy", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + objOriginal := obj.DeepCopy() + + patch := []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, 2)) + Expect(cl.SubResource("scale").Patch(ctx, obj, client.RawPatch(types.MergePatchType, patch))).NotTo(HaveOccurred()) + + actual := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).To(Succeed()) + + objOriginal.APIVersion = actual.APIVersion + objOriginal.Kind = actual.Kind + objOriginal.ResourceVersion = actual.ResourceVersion + objOriginal.Spec.Replicas = ptr.To(int32(2)) + Expect(cmp.Diff(objOriginal, actual)).To(BeEmpty()) + }) + + It("should not change the status of typed objects that have a status subresource on patch", func(ctx SpecContext) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + } + Expect(cl.Create(ctx, obj)).To(Succeed()) + original := obj.DeepCopy() + + obj.Status.Phase = "Running" + Expect(cl.Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Status).To(BeEquivalentTo(corev1.PodStatus{})) + }) + + It("should not change non-status field of typed objects that have a status subresource on status patch", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + objOriginal := obj.DeepCopy() + + obj.Spec.PodCIDR = cidrFromStatusUpdate + obj.Status.NodeInfo.MachineID = "machine-id" + Expect(cl.Status().Patch(ctx, obj, client.MergeFrom(objOriginal))).NotTo(HaveOccurred()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + + objOriginal.APIVersion = actual.APIVersion + objOriginal.Kind = actual.Kind + objOriginal.ResourceVersion = actual.ResourceVersion + objOriginal.Status.NodeInfo.MachineID = "machine-id" + Expect(cmp.Diff(objOriginal, actual)).To(BeEmpty()) + }) + + It("should not change the status of objects with status subresource when creating through apply ", func(ctx SpecContext) { + obj := corev1applyconfigurations. + Pod("node", ""). + WithStatus( + corev1applyconfigurations.PodStatus().WithPhase("Running"), + ) + + cl := NewClientBuilder().WithStatusSubresource(&corev1.Pod{}).Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("test"))).To(Succeed()) + + p := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: *obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(p), p)).To(Succeed()) + + Expect(p.Status).To(BeComparableTo(corev1.PodStatus{})) + }) + + It("should not change the status of objects with status subresource when updating through apply ", func(ctx SpecContext) { + + cl := NewClientBuilder().WithStatusSubresource(&corev1.Pod{}).Build() + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod"}} + Expect(cl.Create(ctx, pod)).NotTo(HaveOccurred()) + + obj := corev1applyconfigurations. + Pod(pod.Name, ""). + WithStatus( + corev1applyconfigurations.PodStatus().WithPhase("Running"), + ) + Expect(cl.Apply(ctx, obj, client.FieldOwner("test"))).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + + Expect(pod.Status).To(BeComparableTo(corev1.PodStatus{})) + }) + + It("should only change status on status apply", func(ctx SpecContext) { + initial := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + } + cl := NewClientBuilder().WithStatusSubresource(&corev1.Node{}).WithObjects(initial).Build() + + ac := corev1applyconfigurations.Node(initial.Name). + WithSpec(corev1applyconfigurations.NodeSpec().WithPodCIDR(initial.Spec.PodCIDR + "-updated")). + WithStatus(corev1applyconfigurations.NodeStatus().WithPhase(corev1.NodeRunning)) + + Expect(cl.Status().Apply(ctx, ac, client.FieldOwner("test-owner"))).To(Succeed()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: initial.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).To(Succeed()) + + initial.ResourceVersion = actual.ResourceVersion + initial.Status = actual.Status + Expect(initial).To(BeComparableTo(actual)) + }) + + It("should Unmarshal the schemaless object with int64 to preserve ints", func(ctx SpecContext) { + schemeBuilder := &scheme.Builder{GroupVersion: schema.GroupVersion{Group: "test", Version: "v1"}} + schemeBuilder.Register(&WithSchemalessSpec{}) + + scheme := runtime.NewScheme() + Expect(schemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) + + spec := Schemaless{ + "key": int64(1), + } + + obj := &WithSchemalessSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-foo", + }, + Spec: spec, + } + cl := NewClientBuilder().WithScheme(scheme).WithStatusSubresource(obj).WithObjects(obj).Build() + + Expect(cl.Update(ctx, obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + }) + + It("should Unmarshal the schemaless object with float64 to preserve ints", func(ctx SpecContext) { + schemeBuilder := &scheme.Builder{GroupVersion: schema.GroupVersion{Group: "test", Version: "v1"}} + schemeBuilder.Register(&WithSchemalessSpec{}) + + scheme := runtime.NewScheme() + Expect(schemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) + + spec := Schemaless{ + "key": 1.1, + } + + obj := &WithSchemalessSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-foo", + }, + Spec: spec, + } + cl := NewClientBuilder().WithScheme(scheme).WithStatusSubresource(obj).WithObjects(obj).Build() + + Expect(cl.Update(ctx, obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + }) + + It("should not change the status of unstructured objects that are configured to have a status subresource on update", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + + err := unstructured.SetNestedField(obj.Object, map[string]any{"state": "old"}, "status") + Expect(err).NotTo(HaveOccurred()) + + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") + Expect(err).ToNot(HaveOccurred()) + + Expect(cl.Update(ctx, obj)).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "old"})) + }) + + It("should not change non-status fields of unstructured objects that are configured to have a status subresource on status update", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + + err := unstructured.SetNestedField(obj.Object, "original", "spec") + Expect(err).NotTo(HaveOccurred()) + + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + err = unstructured.SetNestedField(obj.Object, "from-status-update", "spec") + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") + Expect(err).ToNot(HaveOccurred()) + + Expect(cl.Status().Update(ctx, obj)).To(Succeed()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "new"})) + Expect(obj.Object["spec"]).To(BeEquivalentTo("original")) + }) + + It("should not change the status of known unstructured objects that have a status subresource on update", func(ctx SpecContext) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + // update using unstructured + u := &unstructured.Unstructured{} + u.SetAPIVersion("v1") + u.SetKind("Pod") + u.SetName(obj.Name) + err := cl.Get(ctx, client.ObjectKeyFromObject(u), u) + Expect(err).NotTo(HaveOccurred()) + + err = unstructured.SetNestedField(u.Object, string(corev1.RestartPolicyNever), "spec", "restartPolicy") + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(u.Object, string(corev1.PodRunning), "status", "phase") + Expect(err).NotTo(HaveOccurred()) + + Expect(cl.Update(ctx, u)).To(Succeed()) + + actual := &corev1.Pod{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), actual)).To(Succeed()) + obj.ResourceVersion = actual.ResourceVersion + // only the spec mutation should persist + obj.Spec.RestartPolicy = corev1.RestartPolicyNever + Expect(cmp.Diff(obj, actual)).To(BeEmpty()) + }) + + It("should not change non-status field of known unstructured objects that have a status subresource on status update", func(ctx SpecContext) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + // status update using unstructured + u := &unstructured.Unstructured{} + u.SetAPIVersion("v1") + u.SetKind("Pod") + u.SetName(obj.Name) + err := cl.Get(ctx, client.ObjectKeyFromObject(u), u) + Expect(err).NotTo(HaveOccurred()) + + err = unstructured.SetNestedField(u.Object, string(corev1.RestartPolicyNever), "spec", "restartPolicy") + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(u.Object, string(corev1.PodRunning), "status", "phase") + Expect(err).NotTo(HaveOccurred()) + + Expect(cl.Status().Update(ctx, u)).To(Succeed()) + + actual := &corev1.Pod{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), actual)).To(Succeed()) + obj.ResourceVersion = actual.ResourceVersion + // only the status mutation should persist + obj.Status.Phase = corev1.PodRunning + Expect(cmp.Diff(obj, actual)).To(BeEmpty()) + }) + + It("should not change the status of unstructured objects that are configured to have a status subresource on patch", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + cl := NewClientBuilder().WithStatusSubresource(obj).Build() + + Expect(cl.Create(ctx, obj)).To(Succeed()) + original := obj.DeepCopy() + + err := unstructured.SetNestedField(obj.Object, map[string]interface{}{"count": int64(2)}, "status") + Expect(err).ToNot(HaveOccurred()) + Expect(cl.Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Object["status"]).To(BeNil()) + + }) + + It("should not change non-status fields of unstructured objects that are configured to have a status subresource on status patch", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + + err := unstructured.SetNestedField(obj.Object, "original", "spec") + Expect(err).NotTo(HaveOccurred()) + + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + original := obj.DeepCopy() + + err = unstructured.SetNestedField(obj.Object, "from-status-update", "spec") + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") + Expect(err).ToNot(HaveOccurred()) + + Expect(cl.Status().Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "new"})) + Expect(obj.Object["spec"]).To(BeEquivalentTo("original")) + }) + + It("should return not found on status update of resources that don't have a status subresource", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + + cl := NewClientBuilder().WithObjects(obj).Build() + + err := cl.Status().Update(ctx, obj) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + evictionTypes := []client.Object{ + &policyv1beta1.Eviction{}, + &policyv1.Eviction{}, + } + for _, tp := range evictionTypes { + It("should delete a pod through the eviction subresource", func(ctx SpecContext) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + + cl := NewClientBuilder().WithObjects(pod).Build() + + err := cl.SubResource("eviction").Create(ctx, pod, tp) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Get(ctx, client.ObjectKeyFromObject(pod), pod) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should return not found when attempting to evict a pod that doesn't exist", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + err := cl.SubResource("eviction").Create(ctx, pod, tp) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should return not found when attempting to evict something other than a pod", func(ctx SpecContext) { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + cl := NewClientBuilder().WithObjects(ns).Build() + + err := cl.SubResource("eviction").Create(ctx, ns, tp) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should return an error when using the wrong subresource", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + + err := cl.SubResource("eviction-subresource").Create(ctx, &corev1.Namespace{}, tp) + Expect(err).To(HaveOccurred()) + }) + } + + It("should error when creating an eviction with the wrong type", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + err := cl.SubResource("eviction").Create(ctx, &corev1.Pod{}, &corev1.Namespace{}) + Expect(apierrors.IsBadRequest(err)).To(BeTrue()) + }) + + It("should create a ServiceAccount token through the token subresource", func(ctx SpecContext) { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + cl := NewClientBuilder().WithObjects(sa).Build() + + tokenRequest := &authenticationv1.TokenRequest{} + err := cl.SubResource("token").Create(ctx, sa, tokenRequest) + Expect(err).NotTo(HaveOccurred()) + + Expect(tokenRequest.Status.Token).NotTo(Equal("")) + Expect(tokenRequest.Status.ExpirationTimestamp).NotTo(Equal(metav1.Time{})) + }) + + It("should return not found when creating a token for a ServiceAccount that doesn't exist", func(ctx SpecContext) { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + cl := NewClientBuilder().Build() + + err := cl.SubResource("token").Create(ctx, sa, &authenticationv1.TokenRequest{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should error when creating a token with the wrong subresource type", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + err := cl.SubResource("token").Create(ctx, &corev1.ServiceAccount{}, &corev1.Namespace{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsBadRequest(err)).To(BeTrue()) + }) + + It("should error when creating a token with the wrong type", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + err := cl.SubResource("token").Create(ctx, &corev1.Secret{}, &authenticationv1.TokenRequest{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should leave typemeta empty on typed get", func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + }}).Build() + + var pod corev1.Pod + Expect(cl.Get(ctx, client.ObjectKey{Namespace: "default", Name: "foo"}, &pod)).NotTo(HaveOccurred()) + + Expect(pod.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("should leave typemeta empty on typed list", func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + }}).Build() + + var podList corev1.PodList + Expect(cl.List(ctx, &podList)).NotTo(HaveOccurred()) + Expect(podList.ListMeta).To(Equal(metav1.ListMeta{})) + Expect(podList.Items[0].TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("should allow concurrent patches to a configMap", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + ResourceVersion: "0", + }, + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries) + + for i := range tries { + go func() { + defer wg.Done() + defer GinkgoRecover() + + newObj := obj.DeepCopy() + newObj.Data = map[string]string{"foo": strconv.Itoa(i)} + Expect(cl.Patch(ctx, newObj, client.MergeFrom(obj))).To(Succeed()) + }() + } + wg.Wait() + + // While the order is not deterministic, there must be $tries distinct updates + // that each increment the resource version by one + Expect(cl.Get(ctx, client.ObjectKey{Name: "foo"}, obj)).To(Succeed()) + Expect(obj.ResourceVersion).To(Equal(strconv.Itoa(tries))) + }) + + It("should not allow concurrent patches to a configMap if the patch contains a ResourceVersion", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + ResourceVersion: "0", + }, + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + wg := sync.WaitGroup{} + wg.Add(5) + + for i := range 5 { + go func() { + defer wg.Done() + defer GinkgoRecover() + + newObj := obj.DeepCopy() + newObj.ResourceVersion = "1" // include an invalid RV to cause a conflict + newObj.Data = map[string]string{"foo": strconv.Itoa(i)} + Expect(apierrors.IsConflict(cl.Patch(ctx, newObj, client.MergeFrom(obj)))).To(BeTrue()) + }() + } + wg.Wait() + }) + + It("should allow concurrent updates to an object that allows unconditionalUpdate if the incoming request has no RV", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + ResourceVersion: "0", + }, + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries) + + for i := range tries { + go func() { + defer wg.Done() + defer GinkgoRecover() + + newObj := obj.DeepCopy() + newObj.Data = map[string]string{"foo": strconv.Itoa(i)} + newObj.ResourceVersion = "" + Expect(cl.Update(ctx, newObj)).To(Succeed()) + }() + } + wg.Wait() + + // While the order is not deterministic, there must be $tries distinct updates + // that each increment the resource version by one + Expect(cl.Get(ctx, client.ObjectKey{Name: "foo"}, obj)).To(Succeed()) + Expect(obj.ResourceVersion).To(Equal(strconv.Itoa(tries))) + }) + + It("If a create races with an update for an object that allows createOnUpdate, the update should always succeed", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries * 2) + + for i := range tries { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + go func() { + defer wg.Done() + defer GinkgoRecover() + + // this may or may not succeed depending on if we win the race. Either is acceptable, + // but if it fails, it must fail due to an AlreadyExists. + err := cl.Create(ctx, obj.DeepCopy()) + if err != nil { + Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) + } + }() + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This must always succeed, regardless of the outcome of the create. + Expect(cl.Update(ctx, obj.DeepCopy())).To(Succeed()) + }() + } + + wg.Wait() + }) + + It("If a delete races with an update for an object that allows createOnUpdate, the update should always succeed", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries * 2) + + for i := range tries { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + Expect(cl.Create(ctx, obj.DeepCopy())).To(Succeed()) + + go func() { + defer wg.Done() + defer GinkgoRecover() + + Expect(cl.Delete(ctx, obj.DeepCopy())).To(Succeed()) + }() + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This must always succeed, regardless of if the delete came before or + // after us. + Expect(cl.Update(ctx, obj.DeepCopy())).To(Succeed()) + }() + } + + wg.Wait() + }) + + It("If a DeleteAllOf races with a delete, the DeleteAllOf should always succeed", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const objects = 50 + wg := sync.WaitGroup{} + wg.Add(objects) + + for i := range objects { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + Expect(cl.Create(ctx, obj.DeepCopy())).To(Succeed()) + } + + for i := range objects { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This may or may not succeed depending on if the DeleteAllOf is faster, + // but if it fails, it should be a not found. + err := cl.Delete(ctx, obj) + if err != nil { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } + }() + } + Expect(cl.DeleteAllOf(ctx, &corev1.Service{})).To(Succeed()) + + wg.Wait() + }) + + It("If an update races with a scale update, only one of them succeeds", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(appsv1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const tries = 5000 + for i := range tries { + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + Expect(cl.Create(ctx, dep)).To(Succeed()) + + wg := sync.WaitGroup{} + wg.Add(2) + var updateSucceeded bool + var scaleSucceeded bool + + go func() { + defer wg.Done() + defer GinkgoRecover() + + dep := dep.DeepCopy() + dep.Annotations = map[string]string{"foo": "bar"} + + // This may or may not fail. If it does fail, it must be a conflict. + err := cl.Update(ctx, dep) + if err != nil { + Expect(apierrors.IsConflict(err)).To(BeTrue()) + } else { + updateSucceeded = true + } + }() + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This may or may not fail. If it does fail, it must be a conflict. + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 10}} + err := cl.SubResource("scale").Update(ctx, dep.DeepCopy(), client.WithSubResourceBody(scale)) + if err != nil { + Expect(apierrors.IsConflict(err)).To(BeTrue()) + } else { + scaleSucceeded = true + } + }() + + wg.Wait() + Expect(updateSucceeded).ToNot(Equal(scaleSucceeded)) + } + + }) + + It("disallows scale subresources on unsupported built-in types", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + Expect(apiextensions.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 2}} + expectedErr := "unimplemented scale subresource for resource *v1.Pod" + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scale).Error()).To(Equal(expectedErr)) + Expect(cl.SubResource(subResourceScale).Update(ctx, obj, client.WithSubResourceBody(scale)).Error()).To(Equal(expectedErr)) + }) + + It("disallows scale subresources on non-existing objects", func(ctx SpecContext) { + obj := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](2), + }, + } + cl := NewClientBuilder().Build() + + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 2}} + expectedErr := "deployments.apps \"foo\" not found" + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scale).Error()).To(Equal(expectedErr)) + Expect(cl.SubResource(subResourceScale).Update(ctx, obj, client.WithSubResourceBody(scale)).Error()).To(Equal(expectedErr)) + }) + + It("clears typemeta from structured objects on create", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().Build() + Expect(cl.Create(ctx, obj)).To(Succeed()) + Expect(obj.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on update", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + Expect(cl.Update(ctx, obj)).To(Succeed()) + Expect(obj.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on patch", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + original := obj.DeepCopy() + obj.TypeMeta = metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + } + Expect(cl.Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) + Expect(obj.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on get", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + target := &corev1.ConfigMap{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), target)).To(Succeed()) + Expect(target.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on list", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + target := &corev1.ConfigMapList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + Expect(cl.List(ctx, target)).To(Succeed()) + Expect(target.TypeMeta).To(Equal(metav1.TypeMeta{})) + Expect(target.Items[0].TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("is threadsafe", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + + u := func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("custom/v1") + u.SetKind("Version") + u.SetName("foo") + return u + } + + uList := func() *unstructured.UnstructuredList { + u := &unstructured.UnstructuredList{} + u.SetAPIVersion("custom/v1") + u.SetKind("Version") + + return u + } + + meta := func() *metav1.PartialObjectMetadata { + return &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "custom/v1", + Kind: "Version", + }, + } + } + metaList := func() *metav1.PartialObjectMetadataList { + return &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + + APIVersion: "custom/v1", + Kind: "Version", + }, + } + } + + pod := func() *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }} + } + + ops := []func(){ + func() { _ = cl.Create(ctx, u()) }, + func() { _ = cl.Get(ctx, client.ObjectKeyFromObject(u()), u()) }, + func() { _ = cl.Update(ctx, u()) }, + func() { _ = cl.Patch(ctx, u(), client.RawPatch(types.StrategicMergePatchType, []byte("foo"))) }, + func() { _ = cl.Delete(ctx, u()) }, + func() { _ = cl.DeleteAllOf(ctx, u(), client.HasLabels{"foo"}) }, + func() { _ = cl.List(ctx, uList()) }, + + func() { _ = cl.Create(ctx, meta()) }, + func() { _ = cl.Get(ctx, client.ObjectKeyFromObject(meta()), meta()) }, + func() { _ = cl.Update(ctx, meta()) }, + func() { _ = cl.Patch(ctx, meta(), client.RawPatch(types.StrategicMergePatchType, []byte("foo"))) }, + func() { _ = cl.Delete(ctx, meta()) }, + func() { _ = cl.DeleteAllOf(ctx, meta(), client.HasLabels{"foo"}) }, + func() { _ = cl.List(ctx, metaList()) }, + + func() { _ = cl.Create(ctx, pod()) }, + func() { _ = cl.Get(ctx, client.ObjectKeyFromObject(pod()), pod()) }, + func() { _ = cl.Update(ctx, pod()) }, + func() { _ = cl.Patch(ctx, pod(), client.RawPatch(types.StrategicMergePatchType, []byte("foo"))) }, + func() { _ = cl.Delete(ctx, pod()) }, + func() { _ = cl.DeleteAllOf(ctx, pod(), client.HasLabels{"foo"}) }, + func() { _ = cl.List(ctx, &corev1.PodList{}) }, + } + + wg := sync.WaitGroup{} + wg.Add(len(ops)) + for _, op := range ops { + go func() { + defer wg.Done() + op() + }() + } + + wg.Wait() + }) + + DescribeTable("mutating operations return the updated object", + func(ctx SpecContext, mutate func(ctx SpecContext) (*corev1.ConfigMap, error)) { + mutated, err := mutate(ctx) + Expect(err).NotTo(HaveOccurred()) + + var retrieved corev1.ConfigMap + Expect(cl.Get(ctx, client.ObjectKeyFromObject(mutated), &retrieved)).To(Succeed()) + + Expect(&retrieved).To(BeComparableTo(mutated)) + }, + + Entry("create", func(ctx SpecContext) (*corev1.ConfigMap, error) { + cl = NewClientBuilder().Build() + cm.ResourceVersion = "" + return cm, cl.Create(ctx, cm) + }), + Entry("update", func(ctx SpecContext) (*corev1.ConfigMap, error) { + cl = NewClientBuilder().WithObjects(cm).Build() + cm.Labels = map[string]string{"updated-label": "update-test"} + cm.Data["new-key"] = "new-value" + return cm, cl.Update(ctx, cm) + }), + Entry("patch", func(ctx SpecContext) (*corev1.ConfigMap, error) { + cl = NewClientBuilder().WithObjects(cm).Build() + original := cm.DeepCopy() + + cm.Labels = map[string]string{"updated-label": "update-test"} + cm.Data["new-key"] = "new-value" + return cm, cl.Patch(ctx, cm, client.MergeFrom(original)) + }), + Entry("Create through Apply", func(ctx SpecContext) (*corev1.ConfigMap, error) { + ac := corev1applyconfigurations.ConfigMap(cm.Name, cm.Namespace).WithData(cm.Data) + + cl = NewClientBuilder().Build() + Expect(cl.Apply(ctx, ac, client.FieldOwner("foo"))).To(Succeed()) + + serialized, err := json.Marshal(ac) + Expect(err).NotTo(HaveOccurred()) + + var cm corev1.ConfigMap + Expect(json.Unmarshal(serialized, &cm)).To(Succeed()) + + // ApplyConfigurations always have TypeMeta set as they do not support using the scheme + // to retrieve gvk. + cm.TypeMeta = metav1.TypeMeta{} + return &cm, nil + }), + Entry("Update through Apply", func(ctx SpecContext) (*corev1.ConfigMap, error) { + ac := corev1applyconfigurations.ConfigMap(cm.Name, cm.Namespace). + WithLabels(map[string]string{"updated-label": "update-test"}). + WithData(map[string]string{"new-key": "new-value"}) + + cl = NewClientBuilder().WithObjects(cm).Build() + Expect(cl.Apply(ctx, ac, client.FieldOwner("foo"))).To(Succeed()) + + serialized, err := json.Marshal(ac) + Expect(err).NotTo(HaveOccurred()) + + var cm corev1.ConfigMap + Expect(json.Unmarshal(serialized, &cm)).To(Succeed()) + + // ApplyConfigurations always have TypeMeta set as they do not support using the scheme + // to retrieve gvk. + cm.TypeMeta = metav1.TypeMeta{} + return &cm, nil + }), + ) + + It("supports server-side apply of a client-go resource", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetName("foo") + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "data")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "data")).To(Succeed()) + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"other": "data"})) + }) + + It("supports server-side apply of a custom resource", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("custom/v1") + obj.SetKind("FakeResource") + obj.SetName("foo") + result := obj.DeepCopy() + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "spec")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "spec")).To(Succeed()) + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"other": "data"})) + }) + + It("errors out when doing SSA with managedFields set", func(ctx SpecContext) { + cl := NewClientBuilder().WithReturnManagedFields().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetName("foo") + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "data")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + err := cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo")) //nolint:staticcheck // will be removed once client.Apply is removed + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("metadata.managedFields must be nil")) + }) + + It("supports server-side apply using a custom type converter", func(ctx SpecContext) { + cl := NewClientBuilder(). + WithTypeConverters(clientgoapplyconfigurations.NewTypeConverter(clientgoscheme.Scheme)). + Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetName("foo") + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "data")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "data")).To(Succeed()) + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"other": "data"})) + }) + + It("returns managedFields if configured to do so", func(ctx SpecContext) { + cl := NewClientBuilder().WithReturnManagedFields().Build() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-cm", + Namespace: "default", + }, + Data: map[string]string{ + "initial": "data", + }, + } + Expect(cl.Create(ctx, cm)).NotTo(HaveOccurred()) + Expect(cm.ManagedFields).NotTo(BeNil()) + + retrieved := &corev1.ConfigMap{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), retrieved)).NotTo(HaveOccurred()) + Expect(retrieved.ManagedFields).NotTo(BeNil()) + + cm.Data["another"] = "value" + cm.SetManagedFields(nil) + Expect(cl.Update(ctx, cm)).NotTo(HaveOccurred()) + Expect(cm.ManagedFields).NotTo(BeNil()) + + cm.SetManagedFields(nil) + beforePatch := cm.DeepCopy() + cm.Data["a-third"] = "value" + Expect(cl.Patch(ctx, cm, client.MergeFrom(beforePatch))).NotTo(HaveOccurred()) + Expect(cm.ManagedFields).NotTo(BeNil()) + + u := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": cm.Name, + "namespace": cm.Namespace, + }, + "data": map[string]any{ + "ssa": "value", + }, + }} + Expect(cl.Patch(ctx, u, client.Apply, client.FieldOwner("foo"))).NotTo(HaveOccurred()) //nolint:staticcheck // will be removed once client.Apply is removed + _, exists, err := unstructured.NestedFieldNoCopy(u.Object, "metadata", "managedFields") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + + var list corev1.ConfigMapList + Expect(cl.List(ctx, &list)).NotTo(HaveOccurred()) + for _, item := range list.Items { + Expect(item.ManagedFields).NotTo(BeNil()) + } + }) + + It("clears managedFields from objects in a list", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + Expect(cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + var list corev1.ConfigMapList + Expect(cl.List(ctx, &list)).NotTo(HaveOccurred()) + for _, item := range list.Items { + Expect(item.ManagedFields).To(BeNil()) + } + }) + + It("supports server-side apply of a client-go resource via Apply method", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + Expect(cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}} + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(BeComparableTo(map[string]string{"some": "data"})) + + obj.Data = map[string]string{"other": "data"} + Expect(cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(BeComparableTo(map[string]string{"other": "data"})) + }) + + It("returns a conflict when trying to Create an object with UID set through Apply", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithUID("123") + + err := cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsConflict(err)).To(BeTrue()) + }) + + It("errors when trying to server-side apply an object without configuring a FieldManager", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + err := cl.Apply(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "Expected error to be an invalid error") + }) + + It("errors when trying to server-side apply an object with an invalid FieldManager", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + err := cl.Apply(ctx, obj, client.FieldOwner("\x00")) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "Expected error to be an invalid error") + }) + + It("supports server-side apply of a custom resource via Apply method", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("custom/v1") + obj.SetKind("FakeResource") + obj.SetName("foo") + result := obj.DeepCopy() + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "spec")).To(Succeed()) + + applyConfig := client.ApplyConfigurationFromUnstructured(obj) + Expect(cl.Apply(ctx, applyConfig, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "spec")).To(Succeed()) + applyConfig2 := client.ApplyConfigurationFromUnstructured(obj) + Expect(cl.Apply(ctx, applyConfig2, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"other": "data"})) + }) + + It("sets the fieldManager in create, patch and update", func(ctx SpecContext) { + owner := "test-owner" + cl := client.WithFieldOwner( + NewClientBuilder().WithReturnManagedFields().Build(), + owner, + ) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Data: map[string]string{"method": "create"}, + } + Expect(cl.Create(ctx, obj)).NotTo(HaveOccurred()) + + Expect(obj.ManagedFields).NotTo(BeEmpty()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + + originalObj := obj.DeepCopy() + obj.Data["method"] = "patch" + Expect(cl.Patch(ctx, obj, client.MergeFrom(originalObj))).NotTo(HaveOccurred()) + Expect(obj.ManagedFields).NotTo(BeEmpty()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + + obj.Data["method"] = "update" + Expect(cl.Update(ctx, obj)).NotTo(HaveOccurred()) + Expect(obj.ManagedFields).NotTo(BeEmpty()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + }) + + It("sets the fieldManager when creating through update", func(ctx SpecContext) { + owner := "test-owner" + cl := client.WithFieldOwner( + NewClientBuilder().WithReturnManagedFields().Build(), + owner, + ) + + obj := &corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + Expect(cl.Update(ctx, obj, client.FieldOwner(owner))).NotTo(HaveOccurred()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + }) + + // GH-3267 + It("Doesn't leave stale data when updating an object through SSA", func(ctx SpecContext) { + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + + obj.WithData(map[string]string{"bar": "baz"}) + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + var cms corev1.ConfigMapList + Expect(cl.List(ctx, &cms)).NotTo(HaveOccurred()) + Expect(len(cms.Items)).To(BeEquivalentTo(1)) + }) + + It("sets resourceVersion on SSA create", func(ctx SpecContext) { + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + // Ideally we should only test for it to not be empty, realistically we will + // break ppl if we ever start setting a different value. + Expect(obj.ResourceVersion).To(BeEquivalentTo(ptr.To("1"))) + }) + + It("ignores a passed resourceVersion on SSA create", func(ctx SpecContext) { + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}). + WithResourceVersion("1234") + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + Expect(obj.ResourceVersion).To(BeEquivalentTo(ptr.To("1"))) + }) + + It("allows to set deletionTimestamp on an object during SSA create", func(ctx SpecContext) { + now := metav1.Time{Time: time.Now().Round(time.Second)} + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithDeletionTimestamp(now). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + + Expect(obj.DeletionTimestamp).To(BeEquivalentTo(&now)) + }) + + It("will silently ignore a deletionTimestamp update through SSA", func(ctx SpecContext) { + now := metav1.Time{Time: time.Now().Round(time.Second)} + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithDeletionTimestamp(now). + WithFinalizers("foo.bar"). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + Expect(obj.DeletionTimestamp).To(BeEquivalentTo(&now)) + + later := metav1.Time{Time: now.Add(time.Second)} + obj.DeletionTimestamp = &later + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + Expect(*obj.DeletionTimestamp).To(BeEquivalentTo(now)) + }) + + It("will error out if an object with invalid managedFields is added", func(ctx SpecContext) { + fieldV1Map := map[string]interface{}{ + "f:metadata": map[string]interface{}{ + "f:name": map[string]interface{}{}, + "f:labels": map[string]interface{}{}, + "f:annotations": map[string]interface{}{}, + "f:finalizers": map[string]interface{}{}, + }, + } + fieldV1, err := json.Marshal(fieldV1Map) + Expect(err).NotTo(HaveOccurred()) + + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", + Namespace: "default", + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "my-manager", + Operation: metav1.ManagedFieldsOperationUpdate, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{Raw: fieldV1}, + }}, + }} + + Expect(func() { + NewClientBuilder().WithObjects(obj).Build() + }).To(PanicWith(MatchError(ContainSubstring("invalid managedFields")))) + }) + + It("allows adding an object with managedFields", func(ctx SpecContext) { + fieldV1Map := map[string]interface{}{ + "f:metadata": map[string]interface{}{ + "f:name": map[string]interface{}{}, + "f:labels": map[string]interface{}{}, + "f:annotations": map[string]interface{}{}, + "f:finalizers": map[string]interface{}{}, + }, + } + fieldV1, err := json.Marshal(fieldV1Map) + Expect(err).NotTo(HaveOccurred()) + + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", + Namespace: "default", + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "my-manager", + Operation: metav1.ManagedFieldsOperationUpdate, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{Raw: fieldV1}, + APIVersion: "v1", + }}, + }} + + NewClientBuilder().WithObjects(obj).Build() + }) + + It("allows adding an object with invalid managedFields when not using the FieldManagedObjectTracker", func(ctx SpecContext) { + fieldV1Map := map[string]interface{}{ + "f:metadata": map[string]interface{}{ + "f:name": map[string]interface{}{}, + "f:labels": map[string]interface{}{}, + "f:annotations": map[string]interface{}{}, + "f:finalizers": map[string]interface{}{}, + }, + } + fieldV1, err := json.Marshal(fieldV1Map) + Expect(err).NotTo(HaveOccurred()) + + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", + Namespace: "default", + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "my-manager", + Operation: metav1.ManagedFieldsOperationUpdate, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{Raw: fieldV1}, + }}, + }} + + NewClientBuilder(). + WithObjectTracker(testing.NewObjectTracker( + clientgoscheme.Scheme, + serializer.NewCodecFactory(clientgoscheme.Scheme).UniversalDecoder(), + )). + WithObjects(obj). + Build() + }) + + scalableObjs := []client.Object{ + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](2), + }, + }, + &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: ptr.To[int32](2), + }, + }, + &corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: corev1.ReplicationControllerSpec{ + Replicas: ptr.To[int32](2), + }, + }, + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](2), + }, + }, + } + for _, obj := range scalableObjs { + It(fmt.Sprintf("should be able to Get scale subresources for resource %T", obj), func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(obj).Build() + + scaleActual := &autoscalingv1.Scale{} + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scaleActual)).NotTo(HaveOccurred()) + + scaleExpected := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Name: obj.GetName(), + UID: obj.GetUID(), + ResourceVersion: obj.GetResourceVersion(), + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: 2, + }, + } + Expect(cmp.Diff(scaleExpected, scaleActual)).To(BeEmpty()) + }) + + It(fmt.Sprintf("should be able to Update scale subresources for resource %T", obj), func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(obj).Build() + + scaleExpected := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 3}} + Expect(cl.SubResource(subResourceScale).Update(ctx, obj, client.WithSubResourceBody(scaleExpected))).NotTo(HaveOccurred()) + + objActual := obj.DeepCopyObject().(client.Object) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(objActual), objActual)).To(Succeed()) + + objExpected := obj.DeepCopyObject().(client.Object) + switch expected := objExpected.(type) { + case *appsv1.Deployment: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + case *appsv1.ReplicaSet: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + case *corev1.ReplicationController: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + case *appsv1.StatefulSet: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + } + Expect(cmp.Diff(objExpected, objActual)).To(BeEmpty()) + + scaleActual := &autoscalingv1.Scale{} + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scaleActual)).NotTo(HaveOccurred()) + + // When we called Update, these were derived but we need them now to compare. + scaleExpected.Name = scaleActual.Name + scaleExpected.ResourceVersion = scaleActual.ResourceVersion + Expect(cmp.Diff(scaleExpected, scaleActual)).To(BeEmpty()) + }) + + } +}) + +type Schemaless map[string]interface{} + +type WithSchemalessSpec struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec Schemaless `json:"spec,omitempty"` +} + +func (t *WithSchemalessSpec) DeepCopy() *WithSchemalessSpec { + w := &WithSchemalessSpec{ + ObjectMeta: *t.ObjectMeta.DeepCopy(), + } + w.TypeMeta = metav1.TypeMeta{ + APIVersion: t.APIVersion, + Kind: t.Kind, + } + t.Spec.DeepCopyInto(&w.Spec) + + return w +} + +func (t *WithSchemalessSpec) DeepCopyObject() runtime.Object { + return t.DeepCopy() +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Schemaless) DeepCopyInto(out *Schemaless) { + if *in != nil { + *out = make(Schemaless, len(*in)) + for key := range *in { + (*out)[key] = (*in)[key] + } + } +} + +// DeepCopy copies the receiver, creating a new Schemaless. +func (in *Schemaless) DeepCopy() *Schemaless { + if in == nil { + return nil + } + out := new(Schemaless) + in.DeepCopyInto(out) + return out +} + +var _ = Describe("Fake client builder", func() { + It("panics when an index with the same name and GroupVersionKind is registered twice", func(ctx SpecContext) { + // We need any realistic GroupVersionKind, the choice of apps/v1 Deployment is arbitrary. + cb := NewClientBuilder().WithIndex(&appsv1.Deployment{}, + "test-name", + func(client.Object) []string { return nil }) + + Expect(func() { + cb.WithIndex(&appsv1.Deployment{}, + "test-name", + func(client.Object) []string { return []string{"foo"} }) + }).To(Panic()) + }) + + It("should wrap the fake client with an interceptor when WithInterceptorFuncs is called", func(ctx SpecContext) { + var called bool + cli := NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + called = true + return nil + }, + }).Build() + err := cli.Get(ctx, client.ObjectKey{}, &corev1.Pod{}) + Expect(err).NotTo(HaveOccurred()) + Expect(called).To(BeTrue()) + }) + + It("should panic when calling build more than once", func() { + cb := NewClientBuilder() + anotherCb := cb + cb.Build() + Expect(func() { + anotherCb.Build() + }).To(Panic()) + }) }) diff --git a/pkg/client/fake/doc.go b/pkg/client/fake/doc.go index 7d680690dc..47cad3980d 100644 --- a/pkg/client/fake/doc.go +++ b/pkg/client/fake/doc.go @@ -20,7 +20,7 @@ Package fake provides a fake client for testing. A fake client is backed by its simple object store indexed by GroupVersionResource. You can create a fake client with optional objects. - client := NewFakeClientWithScheme(scheme, initObjs...) // initObjs is a slice of runtime.Object + client := NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() You can invoke the methods defined in the Client interface. @@ -28,12 +28,11 @@ When in doubt, it's almost always better not to use this package and instead use envtest.Environment with a real client and API server. WARNING: ⚠️ Current Limitations / Known Issues with the fake Client ⚠️ -- This client does not have a way to inject specific errors to test handled vs. unhandled errors. -- There is some support for sub resources which can cause issues with tests if you're trying to update - e.g. metadata and status in the same reconcile. -- No OpeanAPI validation is performed when creating or updating objects. -- ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update -operations that rely on these fields will fail, or give false positives. - + - This client does not have a way to inject specific errors to test handled vs. unhandled errors. + - There is some support for sub resources which can cause issues with tests if you're trying to update + e.g. metadata and status in the same reconcile. + - No OpenAPI validation is performed when creating or updating objects. + - ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update + operations that rely on these fields will fail, or give false positives. */ package fake diff --git a/pkg/client/fake/typeconverter.go b/pkg/client/fake/typeconverter.go new file mode 100644 index 0000000000..3cb3a0dc77 --- /dev/null +++ b/pkg/client/fake/typeconverter.go @@ -0,0 +1,60 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/managedfields" + "sigs.k8s.io/structured-merge-diff/v6/typed" +) + +type multiTypeConverter struct { + upstream []managedfields.TypeConverter +} + +func (m multiTypeConverter) ObjectToTyped(r runtime.Object, o ...typed.ValidationOptions) (*typed.TypedValue, error) { + var errs []error + for _, u := range m.upstream { + res, err := u.ObjectToTyped(r, o...) + if err != nil { + errs = append(errs, err) + continue + } + + return res, nil + } + + return nil, fmt.Errorf("failed to convert Object to TypedValue: %w", kerrors.NewAggregate(errs)) +} + +func (m multiTypeConverter) TypedToObject(v *typed.TypedValue) (runtime.Object, error) { + var errs []error + for _, u := range m.upstream { + res, err := u.TypedToObject(v) + if err != nil { + errs = append(errs, err) + continue + } + + return res, nil + } + + return nil, fmt.Errorf("failed to convert TypedValue to Object: %w", kerrors.NewAggregate(errs)) +} diff --git a/pkg/client/fake/typeconverter_test.go b/pkg/client/fake/typeconverter_test.go new file mode 100644 index 0000000000..8acba79f88 --- /dev/null +++ b/pkg/client/fake/typeconverter_test.go @@ -0,0 +1,200 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/managedfields" + "sigs.k8s.io/structured-merge-diff/v6/typed" +) + +var _ = Describe("multiTypeConverter", func() { + Describe("ObjectToTyped", func() { + It("should use first converter when it succeeds", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + testTyped := &typed.TypedValue{} + + firstConverter := &mockTypeConverter{ + objectToTypedResult: testTyped, + } + secondConverter := &mockTypeConverter{ + objectToTypedError: errors.New("second converter should not be called"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testTyped)) + }) + + It("should use second converter when first fails", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + testTyped := &typed.TypedValue{} + + firstConverter := &mockTypeConverter{ + objectToTypedError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + objectToTypedResult: testTyped, + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testTyped)) + }) + + It("should return aggregate error when all converters fail", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + firstConverter := &mockTypeConverter{ + objectToTypedError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + objectToTypedError: errors.New("second converter error"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert Object to Typed")) + Expect(err.Error()).To(ContainSubstring("first converter error")) + Expect(err.Error()).To(ContainSubstring("second converter error")) + }) + + It("should return error when no converters provided", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert Object to Typed")) + }) + }) + + Describe("TypedToObject", func() { + It("should use first converter when it succeeds", func() { + testTyped := &typed.TypedValue{} + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + firstConverter := &mockTypeConverter{ + typedToObjectResult: testObj, + } + secondConverter := &mockTypeConverter{ + typedToObjectError: errors.New("second converter should not be called"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testObj)) + }) + + It("should use second converter when first fails", func() { + testTyped := &typed.TypedValue{} + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + firstConverter := &mockTypeConverter{ + typedToObjectError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + typedToObjectResult: testObj, + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testObj)) + }) + + It("should return aggregate error when all converters fail", func() { + testTyped := &typed.TypedValue{} + + firstConverter := &mockTypeConverter{ + typedToObjectError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + typedToObjectError: errors.New("second converter error"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert TypedValue to Object")) + Expect(err.Error()).To(ContainSubstring("first converter error")) + Expect(err.Error()).To(ContainSubstring("second converter error")) + }) + + It("should return error when no converters provided", func() { + testTyped := &typed.TypedValue{} + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert TypedValue to Object")) + }) + }) +}) + +type mockTypeConverter struct { + objectToTypedResult *typed.TypedValue + objectToTypedError error + + typedToObjectResult runtime.Object + typedToObjectError error +} + +func (m *mockTypeConverter) ObjectToTyped(r runtime.Object, o ...typed.ValidationOptions) (*typed.TypedValue, error) { + return m.objectToTypedResult, m.objectToTypedError +} + +func (m *mockTypeConverter) TypedToObject(v *typed.TypedValue) (runtime.Object, error) { + return m.typedToObjectResult, m.typedToObjectError +} diff --git a/pkg/client/fake/versioned_tracker.go b/pkg/client/fake/versioned_tracker.go new file mode 100644 index 0000000000..bc1eaeb951 --- /dev/null +++ b/pkg/client/fake/versioned_tracker.go @@ -0,0 +1,361 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "bytes" + "errors" + "fmt" + "runtime/debug" + "strconv" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/managedfields" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +var _ testing.ObjectTracker = (*versionedTracker)(nil) + +type versionedTracker struct { + upstream testing.ObjectTracker + scheme *runtime.Scheme + withStatusSubresource sets.Set[schema.GroupVersionKind] + usesFieldManagedObjectTracker bool +} + +func (t versionedTracker) Add(obj runtime.Object) error { + var objects []runtime.Object + if meta.IsListType(obj) { + var err error + objects, err = meta.ExtractList(obj) + if err != nil { + return err + } + } else { + objects = []runtime.Object{obj} + } + for _, obj := range objects { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetDeletionTimestamp() != nil && len(accessor.GetFinalizers()) == 0 { + return fmt.Errorf("refusing to create obj %s with metadata.deletionTimestamp but no finalizers", accessor.GetName()) + } + if accessor.GetResourceVersion() == "" { + // We use a "magic" value of 999 here because this field + // is parsed as uint and and 0 is already used in Update. + // As we can't go lower, go very high instead so this can + // be recognized + accessor.SetResourceVersion(trackerAddResourceVersion) + } + + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + + // If the fieldManager can not decode fields, it will just silently clear them. This is pretty + // much guaranteed not to be what someone that initializes a fake client with objects that + // have them set wants, so validate them here. + // Ref https://github.com/kubernetes/kubernetes/blob/a956ef4862993b825bcd524a19260192ff1da72d/staging/src/k8s.io/apimachinery/pkg/util/managedfields/internal/fieldmanager.go#L105 + if t.usesFieldManagedObjectTracker { + if err := managedfields.ValidateManagedFields(accessor.GetManagedFields()); err != nil { + return fmt.Errorf("invalid managedFields on %T: %w", obj, err) + } + } + if err := t.upstream.Add(obj); err != nil { + return err + } + } + + return nil +} + +func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.CreateOptions) error { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetName() == "" { + gvk, _ := apiutil.GVKForObject(obj, t.scheme) + return apierrors.NewInvalid( + gvk.GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + if accessor.GetResourceVersion() != "" { + return apierrors.NewBadRequest("resourceVersion can not be set for Create requests") + } + accessor.SetResourceVersion("1") + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + if err := t.upstream.Create(gvr, obj, ns, opts...); err != nil { + accessor.SetResourceVersion("") + return err + } + + return nil +} + +func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.UpdateOptions) error { + updateOpts, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + + return t.update(gvr, obj, ns, false, false, updateOpts) +} + +func (t versionedTracker) update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, isStatus, deleting bool, opts metav1.UpdateOptions) error { + gvk, err := apiutil.GVKForObject(obj, t.scheme) + if err != nil { + return err + } + obj, needsCreate, err := t.updateObject(gvr, gvk, obj, ns, isStatus, deleting, allowsCreateOnUpdate(gvk), opts.DryRun) + if err != nil { + return err + } + + if needsCreate { + opts := metav1.CreateOptions{DryRun: opts.DryRun, FieldManager: opts.FieldManager} + return t.Create(gvr, obj, ns, opts) + } + + if obj == nil { // Object was deleted in updateObject + return nil + } + + if u, unstructured := obj.(*unstructured.Unstructured); unstructured { + u.SetGroupVersionKind(gvk) + } + + return t.upstream.Update(gvr, obj, ns, opts) +} + +func (t versionedTracker) Patch(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.PatchOptions) error { + patchOptions, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + + gvk, err := apiutil.GVKForObject(obj, t.scheme) + if err != nil { + return err + } + + // We apply patches using a client-go reaction that ends up calling the trackers Patch. As we can't change + // that reaction, we use the callstack to figure out if this originated from the status client. + isStatus := bytes.Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeSubResourceClient).statusPatch")) + + obj, needsCreate, err := t.updateObject(gvr, gvk, obj, ns, isStatus, false, allowsCreateOnUpdate(gvk), patchOptions.DryRun) + if err != nil { + return err + } + if needsCreate { + opts := metav1.CreateOptions{DryRun: patchOptions.DryRun, FieldManager: patchOptions.FieldManager} + return t.Create(gvr, obj, ns, opts) + } + + if obj == nil { // Object was deleted in updateObject + return nil + } + + return t.upstream.Patch(gvr, obj, ns, patchOptions) +} + +// updateObject performs a number of validations and changes related to +// object updates, such as checking and updating the resourceVersion. +func (t versionedTracker) updateObject( + gvr schema.GroupVersionResource, + gvk schema.GroupVersionKind, + obj runtime.Object, + ns string, + isStatus bool, + deleting bool, + allowCreateOnUpdate bool, + dryRun []string, +) (result runtime.Object, needsCreate bool, _ error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, false, fmt.Errorf("failed to get accessor for object: %w", err) + } + + if accessor.GetName() == "" { + return nil, false, apierrors.NewInvalid( + gvk.GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + + oldObject, err := t.Get(gvr, ns, accessor.GetName()) + if err != nil { + // If the resource is not found and the resource allows create on update, issue a + // create instead. + if apierrors.IsNotFound(err) && allowCreateOnUpdate { + // Pass this info to the caller rather than create, because in the SSA case it + // must be created by calling Apply in the upstream tracker, not Create. + // This is because SSA considers Apply and Non-Apply operations to be different + // even when they use the same fieldManager. This behavior is also observable + // with a real Kubernetes apiserver. + // + // Ref https://kubernetes.slack.com/archives/C0EG7JC6T/p1757868204458989?thread_ts=1757808656.002569&cid=C0EG7JC6T + return obj, true, nil + } + return obj, false, err + } + + if t.withStatusSubresource.Has(gvk) { + if isStatus { // copy everything but status and metadata.ResourceVersion from original object + if err := copyStatusFrom(obj, oldObject); err != nil { + return nil, false, fmt.Errorf("failed to copy non-status field for object with status subresouce: %w", err) + } + passedRV := accessor.GetResourceVersion() + if err := copyFrom(oldObject, obj); err != nil { + return nil, false, fmt.Errorf("failed to restore non-status fields: %w", err) + } + accessor.SetResourceVersion(passedRV) + } else { // copy status from original object + if err := copyStatusFrom(oldObject, obj); err != nil { + return nil, false, fmt.Errorf("failed to copy the status for object with status subresource: %w", err) + } + } + } else if isStatus { + return nil, false, apierrors.NewNotFound(gvr.GroupResource(), accessor.GetName()) + } + + oldAccessor, err := meta.Accessor(oldObject) + if err != nil { + return nil, false, err + } + + // If the new object does not have the resource version set and it allows unconditional update, + // default it to the resource version of the existing resource + if accessor.GetResourceVersion() == "" { + switch { + case allowsUnconditionalUpdate(gvk): + accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) + // This is needed because if the patch explicitly sets the RV to null, the client-go reaction we use + // to apply it and whose output we process here will have it unset. It is not clear why the Kubernetes + // apiserver accepts such a patch, but it does so we just copy that behavior. + // Kubernetes apiserver behavior can be checked like this: + // `kubectl patch configmap foo --patch '{"metadata":{"annotations":{"foo":"bar"},"resourceVersion":null}}' -v=9` + case bytes. + Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeClient).Patch")): + // We apply patches using a client-go reaction that ends up calling the trackers Update. As we can't change + // that reaction, we use the callstack to figure out if this originated from the "fakeClient.Patch" func. + accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) + } + } + + if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() { + return nil, false, apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified")) + } + if oldAccessor.GetResourceVersion() == "" { + oldAccessor.SetResourceVersion("0") + } + intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64) + if err != nil { + return nil, false, fmt.Errorf("can not convert resourceVersion %q to int: %w", oldAccessor.GetResourceVersion(), err) + } + intResourceVersion++ + accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10)) + + if !deleting && !deletionTimestampEqual(accessor, oldAccessor) { + return nil, false, fmt.Errorf("error: Unable to edit %s: metadata.deletionTimestamp field is immutable", accessor.GetName()) + } + + if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 { + return nil, false, t.Delete(gvr, accessor.GetNamespace(), accessor.GetName(), metav1.DeleteOptions{DryRun: dryRun}) + } + + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + return obj, false, err +} + +func (t versionedTracker) Apply(gvr schema.GroupVersionResource, applyConfiguration runtime.Object, ns string, opts ...metav1.PatchOptions) error { + patchOptions, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + gvk, err := apiutil.GVKForObject(applyConfiguration, t.scheme) + if err != nil { + return err + } + isStatus := bytes.Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeSubResourceClient).statusPatch")) + + applyConfiguration, needsCreate, err := t.updateObject(gvr, gvk, applyConfiguration, ns, isStatus, false, true, patchOptions.DryRun) + if err != nil { + return err + } + + if needsCreate { + // https://github.com/kubernetes/kubernetes/blob/81affffa1b8d8079836f4cac713ea8d1b2bbf10f/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go#L606 + accessor, err := meta.Accessor(applyConfiguration) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetUID() != "" { + return apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), fmt.Errorf("uid mismatch: the provided object specified uid %s, and no existing object was found", accessor.GetUID())) + } + + if t.withStatusSubresource.Has(gvk) { + // Clear out status for create, for update this is handled in updateObject + if err := copyStatusFrom(&unstructured.Unstructured{}, applyConfiguration); err != nil { + return err + } + } + } + + if applyConfiguration == nil { // Object was deleted in updateObject + return nil + } + + if isStatus { + // We restore everything but status from the tracker where we don't put GVK + // into the object but it must be set for the ManagedFieldsObjectTracker + applyConfiguration.GetObjectKind().SetGroupVersionKind(gvk) + } + return t.upstream.Apply(gvr, applyConfiguration, ns, opts...) +} + +func (t versionedTracker) Delete(gvr schema.GroupVersionResource, ns, name string, opts ...metav1.DeleteOptions) error { + return t.upstream.Delete(gvr, ns, name, opts...) +} + +func (t versionedTracker) Get(gvr schema.GroupVersionResource, ns, name string, opts ...metav1.GetOptions) (runtime.Object, error) { + return t.upstream.Get(gvr, ns, name, opts...) +} + +func (t versionedTracker) List(gvr schema.GroupVersionResource, gvk schema.GroupVersionKind, ns string, opts ...metav1.ListOptions) (runtime.Object, error) { + return t.upstream.List(gvr, gvk, ns, opts...) +} + +func (t versionedTracker) Watch(gvr schema.GroupVersionResource, ns string, opts ...metav1.ListOptions) (watch.Interface, error) { + return t.upstream.Watch(gvr, ns, opts...) +} diff --git a/pkg/client/fieldowner.go b/pkg/client/fieldowner.go new file mode 100644 index 0000000000..5d9437ba91 --- /dev/null +++ b/pkg/client/fieldowner.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// WithFieldOwner wraps a Client and adds the fieldOwner as the field +// manager to all write requests from this client. If additional [FieldOwner] +// options are specified on methods of this client, the value specified here +// will be overridden. +func WithFieldOwner(c Client, fieldOwner string) Client { + return &clientWithFieldManager{ + owner: fieldOwner, + c: c, + Reader: c, + } +} + +type clientWithFieldManager struct { + owner string + c Client + Reader +} + +func (f *clientWithFieldManager) Create(ctx context.Context, obj Object, opts ...CreateOption) error { + return f.c.Create(ctx, obj, append([]CreateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { + return f.c.Update(ctx, obj, append([]UpdateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { + return f.c.Patch(ctx, obj, patch, append([]PatchOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + return f.c.Apply(ctx, obj, append([]ApplyOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { + return f.c.Delete(ctx, obj, opts...) +} + +func (f *clientWithFieldManager) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { + return f.c.DeleteAllOf(ctx, obj, opts...) +} + +func (f *clientWithFieldManager) Scheme() *runtime.Scheme { return f.c.Scheme() } +func (f *clientWithFieldManager) RESTMapper() meta.RESTMapper { return f.c.RESTMapper() } +func (f *clientWithFieldManager) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return f.c.GroupVersionKindFor(obj) +} +func (f *clientWithFieldManager) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return f.c.IsObjectNamespaced(obj) +} + +func (f *clientWithFieldManager) Status() StatusWriter { + return &subresourceClientWithFieldOwner{ + owner: f.owner, + subresourceWriter: f.c.Status(), + } +} + +func (f *clientWithFieldManager) SubResource(subresource string) SubResourceClient { + c := f.c.SubResource(subresource) + return &subresourceClientWithFieldOwner{ + owner: f.owner, + subresourceWriter: c, + SubResourceReader: c, + } +} + +type subresourceClientWithFieldOwner struct { + owner string + subresourceWriter SubResourceWriter + SubResourceReader +} + +func (f *subresourceClientWithFieldOwner) Create(ctx context.Context, obj Object, subresource Object, opts ...SubResourceCreateOption) error { + return f.subresourceWriter.Create(ctx, obj, subresource, append([]SubResourceCreateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *subresourceClientWithFieldOwner) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + return f.subresourceWriter.Update(ctx, obj, append([]SubResourceUpdateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *subresourceClientWithFieldOwner) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + return f.subresourceWriter.Patch(ctx, obj, patch, append([]SubResourcePatchOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *subresourceClientWithFieldOwner) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + return f.subresourceWriter.Apply(ctx, obj, append([]SubResourceApplyOption{FieldOwner(f.owner)}, opts...)...) +} diff --git a/pkg/client/fieldowner_test.go b/pkg/client/fieldowner_test.go new file mode 100644 index 0000000000..069abbc115 --- /dev/null +++ b/pkg/client/fieldowner_test.go @@ -0,0 +1,180 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client_test + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestWithFieldOwner(t *testing.T) { + calls := 0 + fakeClient := testClient(t, "custom-field-mgr", func() { calls++ }) + wrappedClient := client.WithFieldOwner(fakeClient, "custom-field-mgr") + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + dummyObjectAC := corev1applyconfigurations.Namespace(dummyObj.Name) + + _ = wrappedClient.Create(ctx, dummyObj) + _ = wrappedClient.Update(ctx, dummyObj) + _ = wrappedClient.Patch(ctx, dummyObj, nil) + _ = wrappedClient.Apply(ctx, dummyObjectAC) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.Status().Update(ctx, dummyObj) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil) + _ = wrappedClient.Status().Apply(ctx, dummyObjectAC) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil) + _ = wrappedClient.SubResource("some-subresource").Apply(ctx, dummyObjectAC) + + if expectedCalls := 12; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +func TestWithFieldOwnerOverridden(t *testing.T) { + calls := 0 + + fakeClient := testClient(t, "new-field-manager", func() { calls++ }) + wrappedClient := client.WithFieldOwner(fakeClient, "old-field-manager") + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + dummyObjectAC := corev1applyconfigurations.Namespace(dummyObj.Name) + + _ = wrappedClient.Create(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Update(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Patch(ctx, dummyObj, nil, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Apply(ctx, dummyObjectAC, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Update(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Apply(ctx, dummyObjectAC, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Apply(ctx, dummyObjectAC, client.FieldOwner("new-field-manager")) + + if expectedCalls := 12; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +// testClient is a helper function that checks if calls have the expected field manager, +// and calls the callback function on each intercepted call. +func testClient(t *testing.T, expectedFieldManager string, callback func()) client.Client { + // TODO: we could use the dummyClient in interceptor pkg if we move it to an internal pkg + return fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + callback() + out := &client.CreateOptions{} + for _, f := range opts { + f.ApplyToCreate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + callback() + out := &client.UpdateOptions{} + for _, f := range opts { + f.ApplyToUpdate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + callback() + out := &client.PatchOptions{} + for _, f := range opts { + f.ApplyToPatch(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + callback() + out := &client.SubResourceCreateOptions{} + for _, f := range opts { + f.ApplyToSubResourceCreate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + callback() + out := &client.SubResourceUpdateOptions{} + for _, f := range opts { + f.ApplyToSubResourceUpdate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourcePatch: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + callback() + out := &client.SubResourcePatchOptions{} + for _, f := range opts { + f.ApplyToSubResourcePatch(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + Apply: func(ctx context.Context, c client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + callback() + out := &client.ApplyOptions{} + for _, f := range opts { + f.ApplyToApply(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourceApply: func(ctx context.Context, c client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + callback() + out := &client.SubResourceApplyOptions{} + for _, f := range opts { + f.ApplyToSubResourceApply(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + }).Build() +} diff --git a/pkg/client/fieldvalidation.go b/pkg/client/fieldvalidation.go new file mode 100644 index 0000000000..b0f660854e --- /dev/null +++ b/pkg/client/fieldvalidation.go @@ -0,0 +1,117 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// WithFieldValidation wraps a Client and configures field validation, by +// default, for all write requests from this client. Users can override field +// validation for individual write requests. +// +// This wrapper has no effect on apply requests, as they do not support a +// custom fieldValidation setting, it is always strict. +func WithFieldValidation(c Client, validation FieldValidation) Client { + return &clientWithFieldValidation{ + validation: validation, + client: c, + Reader: c, + } +} + +type clientWithFieldValidation struct { + validation FieldValidation + client Client + Reader +} + +func (c *clientWithFieldValidation) Create(ctx context.Context, obj Object, opts ...CreateOption) error { + return c.client.Create(ctx, obj, append([]CreateOption{c.validation}, opts...)...) +} + +func (c *clientWithFieldValidation) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { + return c.client.Update(ctx, obj, append([]UpdateOption{c.validation}, opts...)...) +} + +func (c *clientWithFieldValidation) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { + return c.client.Patch(ctx, obj, patch, append([]PatchOption{c.validation}, opts...)...) +} + +func (c *clientWithFieldValidation) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + return c.client.Apply(ctx, obj, opts...) +} + +func (c *clientWithFieldValidation) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { + return c.client.Delete(ctx, obj, opts...) +} + +func (c *clientWithFieldValidation) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { + return c.client.DeleteAllOf(ctx, obj, opts...) +} + +func (c *clientWithFieldValidation) Scheme() *runtime.Scheme { return c.client.Scheme() } +func (c *clientWithFieldValidation) RESTMapper() meta.RESTMapper { return c.client.RESTMapper() } +func (c *clientWithFieldValidation) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.client.GroupVersionKindFor(obj) +} + +func (c *clientWithFieldValidation) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.client.IsObjectNamespaced(obj) +} + +func (c *clientWithFieldValidation) Status() StatusWriter { + return &subresourceClientWithFieldValidation{ + validation: c.validation, + subresourceWriter: c.client.Status(), + } +} + +func (c *clientWithFieldValidation) SubResource(subresource string) SubResourceClient { + srClient := c.client.SubResource(subresource) + return &subresourceClientWithFieldValidation{ + validation: c.validation, + subresourceWriter: srClient, + SubResourceReader: srClient, + } +} + +type subresourceClientWithFieldValidation struct { + validation FieldValidation + subresourceWriter SubResourceWriter + SubResourceReader +} + +func (c *subresourceClientWithFieldValidation) Create(ctx context.Context, obj Object, subresource Object, opts ...SubResourceCreateOption) error { + return c.subresourceWriter.Create(ctx, obj, subresource, append([]SubResourceCreateOption{c.validation}, opts...)...) +} + +func (c *subresourceClientWithFieldValidation) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + return c.subresourceWriter.Update(ctx, obj, append([]SubResourceUpdateOption{c.validation}, opts...)...) +} + +func (c *subresourceClientWithFieldValidation) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + return c.subresourceWriter.Patch(ctx, obj, patch, append([]SubResourcePatchOption{c.validation}, opts...)...) +} + +func (c *subresourceClientWithFieldValidation) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + return c.subresourceWriter.Apply(ctx, obj, opts...) +} diff --git a/pkg/client/fieldvalidation_test.go b/pkg/client/fieldvalidation_test.go new file mode 100644 index 0000000000..6e6e9e5d17 --- /dev/null +++ b/pkg/client/fieldvalidation_test.go @@ -0,0 +1,297 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +var _ = Describe("ClientWithFieldValidation", func() { + It("should return errors for invalid fields when using strict validation", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + wrappedClient := client.WithFieldValidation(cl, metav1.FieldValidationStrict) + + baseNode := &unstructured.Unstructured{} + baseNode.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Kind: "Node", + Version: "v1", + }) + baseNode.SetName("client-with-field-validation-test-node") + + validNode := baseNode.DeepCopy() + patch := client.MergeFrom(validNode.DeepCopy()) + + invalidNode := baseNode.DeepCopy() + Expect(unstructured.SetNestedField(invalidNode.Object, "value", "spec", "invalidField")).To(Succeed()) + + invalidStatusNode := baseNode.DeepCopy() + Expect(unstructured.SetNestedField(invalidStatusNode.Object, "value", "status", "invalidStatusField")).To(Succeed()) + + err = wrappedClient.Create(ctx, invalidNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"spec.invalidField\"")) + + err = wrappedClient.Create(ctx, validNode) + Expect(err).ToNot(HaveOccurred()) + + err = wrappedClient.Update(ctx, invalidNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"spec.invalidField\"")) + + err = wrappedClient.Patch(ctx, invalidNode, patch) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"spec.invalidField\"")) + + // Status.Create is not supported on Nodes + + err = wrappedClient.Status().Update(ctx, invalidStatusNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + err = wrappedClient.Status().Patch(ctx, invalidStatusNode, patch) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + // Status.Create is not supported on Nodes + + err = wrappedClient.SubResource("status").Update(ctx, invalidStatusNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + err = wrappedClient.SubResource("status").Patch(ctx, invalidStatusNode, patch) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + invalidApplyConfig := client.ApplyConfigurationFromUnstructured(invalidStatusNode) + err = wrappedClient.Status().Apply(ctx, invalidApplyConfig, client.FieldOwner("test-owner")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("field not declared in schema")) + + err = wrappedClient.SubResource("status").Apply(ctx, invalidApplyConfig, client.FieldOwner("test-owner")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("field not declared in schema")) + }) +}) + +func TestWithStrictFieldValidation(t *testing.T) { + calls := 0 + fakeClient := testFieldValidationClient(t, metav1.FieldValidationStrict, func() { calls++ }) + wrappedClient := client.WithFieldValidation(fakeClient, metav1.FieldValidationStrict) + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + + _ = wrappedClient.Create(ctx, dummyObj) + _ = wrappedClient.Update(ctx, dummyObj) + _ = wrappedClient.Apply(ctx, corev1applyconfigurations.ConfigMap("foo", "bar")) + _ = wrappedClient.Patch(ctx, dummyObj, nil) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.Status().Update(ctx, dummyObj) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil) + _ = wrappedClient.Status().Apply(ctx, corev1applyconfigurations.Namespace(""), nil) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil) + _ = wrappedClient.SubResource("some-subresource").Apply(ctx, corev1applyconfigurations.Namespace(""), nil) + + if expectedCalls := 12; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +func TestWithStrictFieldValidationOverridden(t *testing.T) { + calls := 0 + + fakeClient := testFieldValidationClient(t, metav1.FieldValidationWarn, func() { calls++ }) + wrappedClient := client.WithFieldValidation(fakeClient, metav1.FieldValidationStrict) + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + + _ = wrappedClient.Create(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Update(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Patch(ctx, dummyObj, nil, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Status().Update(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil, client.FieldValidation(metav1.FieldValidationWarn)) + + if expectedCalls := 9; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +// testFieldValidationClient is a helper function that checks if calls have the expected field validation, +// and calls the callback function on each intercepted call. +func testFieldValidationClient(t *testing.T, expectedFieldValidation string, callback func()) client.Client { + // TODO: we could use the dummyClient in interceptor pkg if we move it to an internal pkg + return fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + callback() + out := &client.CreateOptions{} + for _, f := range opts { + f.ApplyToCreate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsCreateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.CreateOptions{} + out.ApplyToCreate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + callback() + out := &client.UpdateOptions{} + for _, f := range opts { + f.ApplyToUpdate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsUpdateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.UpdateOptions{} + out.ApplyToUpdate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + Apply: func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + callback() + return nil + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + callback() + out := &client.PatchOptions{} + for _, f := range opts { + f.ApplyToPatch(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsPatchOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.PatchOptions{} + out.ApplyToPatch(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + callback() + out := &client.SubResourceCreateOptions{} + for _, f := range opts { + f.ApplyToSubResourceCreate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsCreateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.CreateOptions{} + out.ApplyToCreate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + callback() + out := &client.SubResourceUpdateOptions{} + for _, f := range opts { + f.ApplyToSubResourceUpdate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsUpdateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.UpdateOptions{} + out.ApplyToUpdate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourcePatch: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + callback() + out := &client.SubResourcePatchOptions{} + for _, f := range opts { + f.ApplyToSubResourcePatch(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsPatchOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.PatchOptions{} + out.ApplyToPatch(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourceApply: func(ctx context.Context, c client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + callback() + return nil + }, + }).Build() +} diff --git a/pkg/client/interceptor/intercept.go b/pkg/client/interceptor/intercept.go new file mode 100644 index 0000000000..b98af1a693 --- /dev/null +++ b/pkg/client/interceptor/intercept.go @@ -0,0 +1,183 @@ +package interceptor + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Funcs contains functions that are called instead of the underlying client's methods. +type Funcs struct { + Get func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error + List func(ctx context.Context, client client.WithWatch, list client.ObjectList, opts ...client.ListOption) error + Create func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.CreateOption) error + Delete func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteOption) error + DeleteAllOf func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error + Update func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error + Patch func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error + Apply func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error + Watch func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) + SubResource func(client client.WithWatch, subResource string) client.SubResourceClient + SubResourceGet func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error + SubResourceCreate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error + SubResourceUpdate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error + SubResourcePatch func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error + SubResourceApply func(ctx context.Context, client client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error +} + +// NewClient returns a new interceptor client that calls the functions in funcs instead of the underlying client's methods, if they are not nil. +func NewClient(interceptedClient client.WithWatch, funcs Funcs) client.WithWatch { + return interceptor{ + client: interceptedClient, + funcs: funcs, + } +} + +type interceptor struct { + client client.WithWatch + funcs Funcs +} + +var _ client.WithWatch = &interceptor{} + +func (c interceptor) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.client.GroupVersionKindFor(obj) +} + +func (c interceptor) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.client.IsObjectNamespaced(obj) +} + +func (c interceptor) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if c.funcs.Get != nil { + return c.funcs.Get(ctx, c.client, key, obj, opts...) + } + return c.client.Get(ctx, key, obj, opts...) +} + +func (c interceptor) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if c.funcs.List != nil { + return c.funcs.List(ctx, c.client, list, opts...) + } + return c.client.List(ctx, list, opts...) +} + +func (c interceptor) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if c.funcs.Create != nil { + return c.funcs.Create(ctx, c.client, obj, opts...) + } + return c.client.Create(ctx, obj, opts...) +} + +func (c interceptor) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if c.funcs.Delete != nil { + return c.funcs.Delete(ctx, c.client, obj, opts...) + } + return c.client.Delete(ctx, obj, opts...) +} + +func (c interceptor) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if c.funcs.Update != nil { + return c.funcs.Update(ctx, c.client, obj, opts...) + } + return c.client.Update(ctx, obj, opts...) +} + +func (c interceptor) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if c.funcs.Patch != nil { + return c.funcs.Patch(ctx, c.client, obj, patch, opts...) + } + return c.client.Patch(ctx, obj, patch, opts...) +} + +func (c interceptor) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + if c.funcs.Apply != nil { + return c.funcs.Apply(ctx, c.client, obj, opts...) + } + + return c.client.Apply(ctx, obj, opts...) +} + +func (c interceptor) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if c.funcs.DeleteAllOf != nil { + return c.funcs.DeleteAllOf(ctx, c.client, obj, opts...) + } + return c.client.DeleteAllOf(ctx, obj, opts...) +} + +func (c interceptor) Status() client.SubResourceWriter { + return c.SubResource("status") +} + +func (c interceptor) SubResource(subResource string) client.SubResourceClient { + if c.funcs.SubResource != nil { + return c.funcs.SubResource(c.client, subResource) + } + return subResourceInterceptor{ + subResourceName: subResource, + client: c.client, + funcs: c.funcs, + } +} + +func (c interceptor) Scheme() *runtime.Scheme { + return c.client.Scheme() +} + +func (c interceptor) RESTMapper() meta.RESTMapper { + return c.client.RESTMapper() +} + +func (c interceptor) Watch(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + if c.funcs.Watch != nil { + return c.funcs.Watch(ctx, c.client, obj, opts...) + } + return c.client.Watch(ctx, obj, opts...) +} + +type subResourceInterceptor struct { + subResourceName string + client client.Client + funcs Funcs +} + +var _ client.SubResourceClient = &subResourceInterceptor{} + +func (s subResourceInterceptor) Get(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error { + if s.funcs.SubResourceGet != nil { + return s.funcs.SubResourceGet(ctx, s.client, s.subResourceName, obj, subResource, opts...) + } + return s.client.SubResource(s.subResourceName).Get(ctx, obj, subResource, opts...) +} + +func (s subResourceInterceptor) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + if s.funcs.SubResourceCreate != nil { + return s.funcs.SubResourceCreate(ctx, s.client, s.subResourceName, obj, subResource, opts...) + } + return s.client.SubResource(s.subResourceName).Create(ctx, obj, subResource, opts...) +} + +func (s subResourceInterceptor) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if s.funcs.SubResourceUpdate != nil { + return s.funcs.SubResourceUpdate(ctx, s.client, s.subResourceName, obj, opts...) + } + return s.client.SubResource(s.subResourceName).Update(ctx, obj, opts...) +} + +func (s subResourceInterceptor) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + if s.funcs.SubResourcePatch != nil { + return s.funcs.SubResourcePatch(ctx, s.client, s.subResourceName, obj, patch, opts...) + } + return s.client.SubResource(s.subResourceName).Patch(ctx, obj, patch, opts...) +} + +func (s subResourceInterceptor) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + if s.funcs.SubResourceApply != nil { + return s.funcs.SubResourceApply(ctx, s.client, s.subResourceName, obj, opts...) + } + return s.client.SubResource(s.subResourceName).Apply(ctx, obj, opts...) +} diff --git a/pkg/client/interceptor/intercept_test.go b/pkg/client/interceptor/intercept_test.go new file mode 100644 index 0000000000..fb58dfeac1 --- /dev/null +++ b/pkg/client/interceptor/intercept_test.go @@ -0,0 +1,443 @@ +package interceptor + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("NewClient", func() { + wrappedClient := dummyClient{} + It("should call the provided Get function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + called = true + return nil + }, + }) + _ = client.Get(ctx, types.NamespacedName{}, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Get function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.Get(ctx, types.NamespacedName{}, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided List function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + List: func(ctx context.Context, client client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + called = true + return nil + }, + }) + _ = client.List(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided List function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + List: func(ctx context.Context, client client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.List(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Apply function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Apply: func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + called = true + return nil + }, + }) + _ = client.Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Apply function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Apply: func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Create function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Create: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + called = true + return nil + }, + }) + _ = client.Create(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Create function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Create: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.Create(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Delete function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Delete: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + called = true + return nil + }, + }) + _ = client.Delete(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Delete function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Delete: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.Delete(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided DeleteAllOf function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + DeleteAllOf: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { + called = true + return nil + }, + }) + _ = client.DeleteAllOf(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided DeleteAllOf function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + DeleteAllOf: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.DeleteAllOf(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Update function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Update: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + called = true + return nil + }, + }) + _ = client.Update(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Update function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Update: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.Update(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Patch function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Patch: func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + called = true + return nil + }, + }) + _ = client.Patch(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Patch function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Patch: func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.Patch(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Watch function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Watch: func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + called = true + return nil, nil + }, + }) + _, _ = client.Watch(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Watch function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Watch: func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + called = true + return nil, nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _, _ = client2.Watch(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided SubResource function", func() { + var called bool + client := NewClient(wrappedClient, Funcs{ + SubResource: func(client client.WithWatch, subResource string) client.SubResourceClient { + called = true + return nil + }, + }) + _ = client.SubResource("") + Expect(called).To(BeTrue()) + }) + It("should call the provided SubResource function with 'status' when calling Status()", func() { + var called bool + client := NewClient(wrappedClient, Funcs{ + SubResource: func(client client.WithWatch, subResource string) client.SubResourceClient { + if subResource == "status" { + called = true + } + return nil + }, + }) + _ = client.Status() + Expect(called).To(BeTrue()) + }) +}) + +var _ = Describe("NewSubResourceClient", func() { + c := dummyClient{} + It("should call the provided Get function", func(ctx SpecContext) { + var called bool + c := NewClient(c, Funcs{ + SubResourceGet: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + _ = c.SubResource("foo").Get(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Get function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(c, Funcs{ + SubResourceGet: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.SubResource("foo").Get(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Update function", func(ctx SpecContext) { + var called bool + client := NewClient(c, Funcs{ + SubResourceUpdate: func(_ context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + _ = client.SubResource("foo").Update(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Update function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(c, Funcs{ + SubResourceUpdate: func(_ context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.SubResource("foo").Update(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Patch function", func(ctx SpecContext) { + var called bool + client := NewClient(c, Funcs{ + SubResourcePatch: func(_ context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + _ = client.SubResource("foo").Patch(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Patch function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(c, Funcs{ + SubResourcePatch: func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.SubResource("foo").Patch(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Create function", func(ctx SpecContext) { + var called bool + client := NewClient(c, Funcs{ + SubResourceCreate: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceCreateOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + _ = client.SubResource("foo").Create(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Create function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(c, Funcs{ + SubResourceCreate: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceCreateOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.SubResource("foo").Create(ctx, nil, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Apply function", func(ctx SpecContext) { + var called bool + client := NewClient(c, Funcs{ + SubResourceApply: func(_ context.Context, client client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + _ = client.SubResource("foo").Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Apply function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(c, Funcs{ + SubResourceApply: func(_ context.Context, client client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.SubResource("foo").Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) +}) + +type dummyClient struct{} + +var _ client.WithWatch = &dummyClient{} + +func (d dummyClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return nil +} + +func (d dummyClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + return nil +} + +func (d dummyClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return nil +} + +func (d dummyClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return nil +} + +func (d dummyClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return nil +} + +func (d dummyClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return nil +} + +func (d dummyClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + return nil +} + +func (d dummyClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + return nil +} + +func (d dummyClient) Status() client.SubResourceWriter { + return d.SubResource("status") +} + +func (d dummyClient) SubResource(subResource string) client.SubResourceClient { + return nil +} + +func (d dummyClient) Scheme() *runtime.Scheme { + return nil +} + +func (d dummyClient) RESTMapper() meta.RESTMapper { + return nil +} + +func (d dummyClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return schema.GroupVersionKind{}, nil +} + +func (d dummyClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return false, nil +} + +func (d dummyClient) Watch(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + return nil, nil +} diff --git a/pkg/config/config_suite_test.go b/pkg/client/interceptor/interceptor_suite_test.go similarity index 63% rename from pkg/config/config_suite_test.go rename to pkg/client/interceptor/interceptor_suite_test.go index 9a494dafbc..08d9fe2281 100644 --- a/pkg/config/config_suite_test.go +++ b/pkg/client/interceptor/interceptor_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,19 +14,23 @@ See the License for the specific language governing permissions and limitations under the License. */ -package config_test +package interceptor import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" ) -func TestScheme(t *testing.T) { +func TestInterceptor(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Config Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Fake client Suite") } + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) +}) diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 58c2ece15b..1af1f3a368 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -20,6 +20,7 @@ import ( "context" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" @@ -50,7 +51,7 @@ type Reader interface { // Get retrieves an obj for the given object key from the Kubernetes Cluster. // obj must be a struct pointer so that obj can be updated with the response // returned by the Server. - Get(ctx context.Context, key ObjectKey, obj Object) error + Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error // List retrieves list of objects for a given namespace and list options. On a // successful call, Items field in the list will be populated with the @@ -60,7 +61,11 @@ type Reader interface { // Writer knows how to create, delete, and update Kubernetes objects. type Writer interface { - // Create saves the object obj in the Kubernetes cluster. + // Apply applies the given apply configuration to the Kubernetes cluster. + Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error + + // Create saves the object obj in the Kubernetes cluster. obj must be a + // struct pointer so that obj can be updated with the content returned by the Server. Create(ctx context.Context, obj Object, opts ...CreateOption) error // Delete deletes the given obj from Kubernetes cluster. @@ -81,20 +86,84 @@ type Writer interface { // StatusClient knows how to create a client which can update status subresource // for kubernetes objects. type StatusClient interface { - Status() StatusWriter + Status() SubResourceWriter +} + +// SubResourceClientConstructor knows how to create a client which can update subresource +// for kubernetes objects. +type SubResourceClientConstructor interface { + // SubResourceClientConstructor returns a subresource client for the named subResource. Known + // upstream subResources usages are: + // - ServiceAccount token creation: + // sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // token := &authenticationv1.TokenRequest{} + // c.SubResource("token").Create(ctx, sa, token) + // + // - Pod eviction creation: + // pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // c.SubResource("eviction").Create(ctx, pod, &policyv1.Eviction{}) + // + // - Pod binding creation: + // pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // binding := &corev1.Binding{Target: corev1.ObjectReference{Name: "my-node"}} + // c.SubResource("binding").Create(ctx, pod, binding) + // + // - CertificateSigningRequest approval: + // csr := &certificatesv1.CertificateSigningRequest{ + // ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + // Status: certificatesv1.CertificateSigningRequestStatus{ + // Conditions: []certificatesv1.[]CertificateSigningRequestCondition{{ + // Type: certificatesv1.CertificateApproved, + // Status: corev1.ConditionTrue, + // }}, + // }, + // } + // c.SubResource("approval").Update(ctx, csr) + // + // - Scale retrieval: + // dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // scale := &autoscalingv1.Scale{} + // c.SubResource("scale").Get(ctx, dep, scale) + // + // - Scale update: + // dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 2}} + // c.SubResource("scale").Update(ctx, dep, client.WithSubResourceBody(scale)) + SubResource(subResource string) SubResourceClient } -// StatusWriter knows how to update status subresource of a Kubernetes object. -type StatusWriter interface { +// StatusWriter is kept for backward compatibility. +type StatusWriter = SubResourceWriter + +// SubResourceReader knows how to read SubResources +type SubResourceReader interface { + Get(ctx context.Context, obj Object, subResource Object, opts ...SubResourceGetOption) error +} + +// SubResourceWriter knows how to update subresource of a Kubernetes object. +type SubResourceWriter interface { + // Create saves the subResource object in the Kubernetes cluster. obj must be a + // struct pointer so that obj can be updated with the content returned by the Server. + Create(ctx context.Context, obj Object, subResource Object, opts ...SubResourceCreateOption) error + // Update updates the fields corresponding to the status subresource for the // given obj. obj must be a struct pointer so that obj can be updated // with the content returned by the Server. - Update(ctx context.Context, obj Object, opts ...UpdateOption) error + Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error // Patch patches the given object's subresource. obj must be a struct // pointer so that obj can be updated with the content returned by the // Server. - Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error + Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error + + // Apply applies the given apply configurations subresource. + Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error +} + +// SubResourceClient knows how to perform CRU operations on Kubernetes objects. +type SubResourceClient interface { + SubResourceReader + SubResourceWriter } // Client knows how to perform CRUD operations on Kubernetes objects. @@ -102,11 +171,16 @@ type Client interface { Reader Writer StatusClient + SubResourceClientConstructor // Scheme returns the scheme this client is using. Scheme() *runtime.Scheme // RESTMapper returns the rest this client is using. RESTMapper() meta.RESTMapper + // GroupVersionKindFor returns the GroupVersionKind for the given object. + GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) + // IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. + IsObjectNamespaced(obj runtime.Object) (bool, error) } // WithWatch supports Watch on top of the CRUD operations supported by @@ -125,7 +199,7 @@ type IndexerFunc func(Object) []string // FieldIndexer knows how to index over a particular "field" such that it // can later be used by a field selector. type FieldIndexer interface { - // IndexFields adds an index with the given field name on the given object type + // IndexField adds an index with the given field name on the given object type // by using the given function to extract the value for that field. If you want // compatibility with the Kubernetes API server, only return one key, and only use // fields that the API server supports. Otherwise, you can return multiple keys, @@ -143,3 +217,13 @@ func IgnoreNotFound(err error) error { } return err } + +// IgnoreAlreadyExists returns nil on AlreadyExists errors. +// All other values that are not AlreadyExists errors or nil are returned unmodified. +func IgnoreAlreadyExists(err error) error { + if apierrors.IsAlreadyExists(err) { + return nil + } + + return err +} diff --git a/pkg/client/metadata_client.go b/pkg/client/metadata_client.go index 59747463a4..d0c6b8e13a 100644 --- a/pkg/client/metadata_client.go +++ b/pkg/client/metadata_client.go @@ -116,7 +116,7 @@ func (mc *metadataClient) Patch(ctx context.Context, obj Object, patch Patch, op } // Get implements client.Client. -func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object) error { +func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { metadata, ok := obj.(*metav1.PartialObjectMetadata) if !ok { return fmt.Errorf("metadata client did not understand object: %T", obj) @@ -124,12 +124,15 @@ func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object) er gvk := metadata.GroupVersionKind() + getOpts := GetOptions{} + getOpts.ApplyOptions(opts) + resInt, err := mc.getResourceInterface(gvk, key.Namespace) if err != nil { return err } - res, err := resInt.Get(ctx, key.Name, metav1.GetOptions{}) + res, err := resInt.Get(ctx, key.Name, *getOpts.AsGetOptions()) if err != nil { return err } @@ -146,9 +149,7 @@ func (mc *metadataClient) List(ctx context.Context, obj ObjectList, opts ...List } gvk := metadata.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") listOpts := ListOptions{} listOpts.ApplyOptions(opts) @@ -167,7 +168,7 @@ func (mc *metadataClient) List(ctx context.Context, obj ObjectList, opts ...List return nil } -func (mc *metadataClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { +func (mc *metadataClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error { metadata, ok := obj.(*metav1.PartialObjectMetadata) if !ok { return fmt.Errorf("metadata client did not understand object: %T", obj) @@ -179,16 +180,24 @@ func (mc *metadataClient) PatchStatus(ctx context.Context, obj Object, patch Pat return err } - data, err := patch.Data(obj) + patchOpts := &SubResourcePatchOptions{} + patchOpts.ApplyOptions(opts) + + body := obj + if patchOpts.SubResourceBody != nil { + body = patchOpts.SubResourceBody + } + + data, err := patch.Data(body) if err != nil { return err } - patchOpts := &PatchOptions{} - res, err := resInt.Patch(ctx, metadata.Name, patch.Type(), data, *patchOpts.AsPatchOptions(), "status") + res, err := resInt.Patch(ctx, metadata.Name, patch.Type(), data, *patchOpts.AsPatchOptions(), subResource) if err != nil { return err } + *metadata = *res metadata.SetGroupVersionKind(gvk) // restore the GVK, which isn't set on metadata return nil diff --git a/pkg/client/namespaced_client.go b/pkg/client/namespaced_client.go index 557598727c..445e91b98b 100644 --- a/pkg/client/namespaced_client.go +++ b/pkg/client/namespaced_client.go @@ -19,10 +19,13 @@ package client import ( "context" "fmt" + "reflect" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/internal/objectutil" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) // NewNamespacedClient wraps an existing client enforcing the namespace value. @@ -52,11 +55,21 @@ func (n *namespacedClient) RESTMapper() meta.RESTMapper { return n.client.RESTMapper() } -// Create implements clinet.Client. +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (n *namespacedClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return n.client.GroupVersionKindFor(obj) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (n *namespacedClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return n.client.IsObjectNamespaced(obj) +} + +// Create implements client.Client. func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper()) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -72,9 +85,9 @@ func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...Creat // Update implements client.Client. func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper()) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -90,9 +103,9 @@ func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...Updat // Delete implements client.Client. func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper()) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -108,9 +121,9 @@ func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...Delet // DeleteAllOf implements client.Client. func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper()) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } if isNamespaceScoped { @@ -121,9 +134,9 @@ func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ... // Patch implements client.Client. func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper()) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -137,19 +150,73 @@ func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, o return n.client.Patch(ctx, obj, patch, opts...) } +func (n *namespacedClient) setNamespaceForApplyConfigIfNamespaceScoped(obj runtime.ApplyConfiguration) error { + var gvk schema.GroupVersionKind + switch o := obj.(type) { + case applyConfiguration: + var err error + gvk, err = gvkFromApplyConfiguration(o) + if err != nil { + return err + } + case *unstructuredApplyConfiguration: + gvk = o.GroupVersionKind() + default: + return fmt.Errorf("object %T is not a valid apply configuration", obj) + } + isNamespaceScoped, err := apiutil.IsGVKNamespaced(gvk, n.RESTMapper()) + if err != nil { + return fmt.Errorf("error finding the scope of the object: %w", err) + } + if isNamespaceScoped { + switch o := obj.(type) { + case applyConfiguration: + if o.GetNamespace() != nil && *o.GetNamespace() != "" && *o.GetNamespace() != n.namespace { + return fmt.Errorf("namespace %s provided for the object %s does not match the namespace %s on the client", + *o.GetNamespace(), ptr.Deref(o.GetName(), ""), n.namespace) + } + v := reflect.ValueOf(o) + withNamespace := v.MethodByName("WithNamespace") + if !withNamespace.IsValid() { + return fmt.Errorf("ApplyConfiguration %T does not have a WithNamespace method", o) + } + if tp := withNamespace.Type(); tp.NumIn() != 1 || tp.In(0).Kind() != reflect.String { + return fmt.Errorf("WithNamespace method of ApplyConfiguration %T must take a single string argument", o) + } + withNamespace.Call([]reflect.Value{reflect.ValueOf(n.namespace)}) + case *unstructuredApplyConfiguration: + if o.GetNamespace() != "" && o.GetNamespace() != n.namespace { + return fmt.Errorf("namespace %s provided for the object %s does not match the namespace %s on the client", + o.GetNamespace(), o.GetName(), n.namespace) + } + o.SetNamespace(n.namespace) + } + } + + return nil +} + +func (n *namespacedClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + if err := n.setNamespaceForApplyConfigIfNamespaceScoped(obj); err != nil { + return err + } + + return n.client.Apply(ctx, obj, opts...) +} + // Get implements client.Client. -func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, n.Scheme(), n.RESTMapper()) +func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } if isNamespaceScoped { if key.Namespace != "" && key.Namespace != n.namespace { - return fmt.Errorf("namespace %s provided for the object %s does not match the namesapce %s on the client", key.Namespace, obj.GetName(), n.namespace) + return fmt.Errorf("namespace %s provided for the object %s does not match the namespace %s on the client", key.Namespace, obj.GetName(), n.namespace) } key.Namespace = n.namespace } - return n.client.Get(ctx, key, obj) + return n.client.Get(ctx, key, obj, opts...) } // List implements client.Client. @@ -161,53 +228,101 @@ func (n *namespacedClient) List(ctx context.Context, obj ObjectList, opts ...Lis } // Status implements client.StatusClient. -func (n *namespacedClient) Status() StatusWriter { - return &namespacedClientStatusWriter{StatusClient: n.client.Status(), namespace: n.namespace, namespacedclient: n} +func (n *namespacedClient) Status() SubResourceWriter { + return n.SubResource("status") } -// ensure namespacedClientStatusWriter implements client.StatusWriter. -var _ StatusWriter = &namespacedClientStatusWriter{} +// SubResource implements client.SubResourceClient. +func (n *namespacedClient) SubResource(subResource string) SubResourceClient { + return &namespacedClientSubResourceClient{ + client: n.client.SubResource(subResource), + namespacedclient: n, + } +} -type namespacedClientStatusWriter struct { - StatusClient StatusWriter - namespace string - namespacedclient Client +// ensure namespacedClientSubResourceClient implements client.SubResourceClient. +var _ SubResourceClient = &namespacedClientSubResourceClient{} + +type namespacedClientSubResourceClient struct { + client SubResourceClient + namespacedclient *namespacedClient } -// Update implements client.StatusWriter. -func (nsw *namespacedClientStatusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper()) +func (nsw *namespacedClientSubResourceClient) Get(ctx context.Context, obj, subResource Object, opts ...SubResourceGetOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) + if err != nil { + return fmt.Errorf("error finding the scope of the object: %w", err) + } + + objectNamespace := obj.GetNamespace() + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) + } + if isNamespaceScoped && objectNamespace == "" { + obj.SetNamespace(nsw.namespacedclient.namespace) + } + + return nsw.client.Get(ctx, obj, subResource, opts...) +} + +func (nsw *namespacedClientSubResourceClient) Create(ctx context.Context, obj, subResource Object, opts ...SubResourceCreateOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() - if objectNamespace != nsw.namespace && objectNamespace != "" { - return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) } if isNamespaceScoped && objectNamespace == "" { - obj.SetNamespace(nsw.namespace) + obj.SetNamespace(nsw.namespacedclient.namespace) } - return nsw.StatusClient.Update(ctx, obj, opts...) + + return nsw.client.Create(ctx, obj, subResource, opts...) } -// Patch implements client.StatusWriter. -func (nsw *namespacedClientStatusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - isNamespaceScoped, err := objectutil.IsAPINamespaced(obj, nsw.namespacedclient.Scheme(), nsw.namespacedclient.RESTMapper()) +// Update implements client.SubResourceWriter. +func (nsw *namespacedClientSubResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) + if err != nil { + return fmt.Errorf("error finding the scope of the object: %w", err) + } + + objectNamespace := obj.GetNamespace() + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) + } + if isNamespaceScoped && objectNamespace == "" { + obj.SetNamespace(nsw.namespacedclient.namespace) + } + return nsw.client.Update(ctx, obj, opts...) +} + +// Patch implements client.SubResourceWriter. +func (nsw *namespacedClientSubResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() - if objectNamespace != nsw.namespace && objectNamespace != "" { - return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) } if isNamespaceScoped && objectNamespace == "" { - obj.SetNamespace(nsw.namespace) + obj.SetNamespace(nsw.namespacedclient.namespace) + } + return nsw.client.Patch(ctx, obj, patch, opts...) +} + +func (nsw *namespacedClientSubResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + if err := nsw.namespacedclient.setNamespaceForApplyConfigIfNamespaceScoped(obj); err != nil { + return err } - return nsw.StatusClient.Patch(ctx, obj, patch, opts...) + return nsw.client.Apply(ctx, obj, opts...) } diff --git a/pkg/client/namespaced_client_test.go b/pkg/client/namespaced_client_test.go index 5b8f3388c8..deae881d4a 100644 --- a/pkg/client/namespaced_client_test.go +++ b/pkg/client/namespaced_client_test.go @@ -22,24 +22,30 @@ import ( "fmt" "sync/atomic" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - rbacv1 "k8s.io/api/rbac/v1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + appsv1applyconfigurations "k8s.io/client-go/applyconfigurations/apps/v1" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + metav1applyconfigurations "k8s.io/client-go/applyconfigurations/meta/v1" + rbacv1applyconfigurations "k8s.io/client-go/applyconfigurations/rbac/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("NamespacedClient", func() { var dep *appsv1.Deployment + var acDep *appsv1applyconfigurations.DeploymentApplyConfiguration var ns = "default" - ctx := context.Background() var count uint64 = 0 var replicaCount int32 = 2 @@ -47,9 +53,9 @@ var _ = Describe("NamespacedClient", func() { var sch = runtime.NewScheme() err := rbacv1.AddToScheme(sch) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) err = appsv1.AddToScheme(sch) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) nonNamespacedClient, err := client.New(cfg, client.Options{Scheme: sch}) Expect(err).NotTo(HaveOccurred()) @@ -75,20 +81,36 @@ var _ = Describe("NamespacedClient", func() { }, }, } + acDep = appsv1applyconfigurations.Deployment(dep.Name, ""). + WithLabels(dep.Labels). + WithSpec(appsv1applyconfigurations.DeploymentSpec(). + WithReplicas(*dep.Spec.Replicas). + WithSelector(metav1applyconfigurations.LabelSelector().WithMatchLabels(dep.Spec.Selector.MatchLabels)). + WithTemplate(corev1applyconfigurations.PodTemplateSpec(). + WithLabels(dep.Spec.Template.Labels). + WithSpec(corev1applyconfigurations.PodSpec(). + WithContainers(corev1applyconfigurations.Container(). + WithName(dep.Spec.Template.Spec.Containers[0].Name). + WithImage(dep.Spec.Template.Spec.Containers[0].Image), + ), + ), + ), + ) + }) Describe("Get", func() { - - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { var err error dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully Get a namespace-scoped object", func() { + + It("should successfully Get a namespace-scoped object", func(ctx SpecContext) { name := types.NamespacedName{Name: dep.Name} result := &appsv1.Deployment{} @@ -97,7 +119,7 @@ var _ = Describe("NamespacedClient", func() { }) It("should error when namespace provided in the object is different than the one "+ - "specified in client", func() { + "specified in client", func(ctx SpecContext) { name := types.NamespacedName{Name: dep.Name, Namespace: "non-default"} result := &appsv1.Deployment{} @@ -106,17 +128,17 @@ var _ = Describe("NamespacedClient", func() { }) Describe("List", func() { - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { var err error dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully List objects when namespace is not specified with the object", func() { + It("should successfully List objects when namespace is not specified with the object", func(ctx SpecContext) { result := &appsv1.DeploymentList{} opts := client.MatchingLabels(dep.Labels) @@ -125,7 +147,7 @@ var _ = Describe("NamespacedClient", func() { Expect(result.Items[0]).To(BeEquivalentTo(*dep)) }) - It("should List objects from the namespace specified in the client", func() { + It("should List objects from the namespace specified in the client", func(ctx SpecContext) { result := &appsv1.DeploymentList{} opts := client.InNamespace("non-default") @@ -135,12 +157,72 @@ var _ = Describe("NamespacedClient", func() { }) }) + Describe("Apply", func() { + AfterEach(func(ctx SpecContext) { + deleteDeployment(ctx, dep, ns) + }) + + It("should successfully apply an object in the right namespace", func(ctx SpecContext) { + err := getClient().Apply(ctx, acDep, client.FieldOwner("test")) + Expect(err).NotTo(HaveOccurred()) + + res, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.GetNamespace()).To(BeEquivalentTo(ns)) + }) + + It("should successfully apply an object in the right namespace through unstructured", func(ctx SpecContext) { + serialized, err := json.Marshal(acDep) + Expect(err).NotTo(HaveOccurred()) + u := &unstructured.Unstructured{} + Expect(json.Unmarshal(serialized, &u.Object)).To(Succeed()) + err = getClient().Apply(ctx, client.ApplyConfigurationFromUnstructured(u), client.FieldOwner("test")) + Expect(err).NotTo(HaveOccurred()) + + res, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.GetNamespace()).To(BeEquivalentTo(ns)) + }) + + It("should not create an object if the namespace of the object is different", func(ctx SpecContext) { + acDep.WithNamespace("non-default") + err := getClient().Apply(ctx, acDep, client.FieldOwner("test")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not match the namespace")) + }) + + It("should not create an object through unstructured if the namespace of the object is different", func(ctx SpecContext) { + acDep.WithNamespace("non-default") + serialized, err := json.Marshal(acDep) + Expect(err).NotTo(HaveOccurred()) + u := &unstructured.Unstructured{} + Expect(json.Unmarshal(serialized, &u.Object)).To(Succeed()) + err = getClient().Apply(ctx, client.ApplyConfigurationFromUnstructured(u), client.FieldOwner("test")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not match the namespace")) + }) + + It("should create a cluster scoped object", func(ctx SpecContext) { + cr := rbacv1applyconfigurations.ClusterRole(fmt.Sprintf("clusterRole-%v", count)) + + err := getClient().Apply(ctx, cr, client.FieldOwner("test")) + Expect(err).NotTo(HaveOccurred()) + + By("checking if the object was created") + res, err := clientset.RbacV1().ClusterRoles().Get(ctx, *cr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res).NotTo(BeNil()) + + deleteClusterRole(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: *cr.Name}}) + }) + }) + Describe("Create", func() { - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully create object in the right namespace", func() { + It("should successfully create object in the right namespace", func(ctx SpecContext) { By("creating the object initially") err := getClient().Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) @@ -151,13 +233,13 @@ var _ = Describe("NamespacedClient", func() { Expect(res.GetNamespace()).To(BeEquivalentTo(ns)) }) - It("should not create object if the namespace of the object is different", func() { + It("should not create object if the namespace of the object is different", func(ctx SpecContext) { By("creating the object initially") dep.SetNamespace("non-default") err := getClient().Create(ctx, dep) Expect(err).To(HaveOccurred()) }) - It("should create a cluster scoped object", func() { + It("should create a cluster scoped object", func(ctx SpecContext) { cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("clusterRole-%v", count), @@ -186,16 +268,16 @@ var _ = Describe("NamespacedClient", func() { Describe("Update", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) dep.Annotations = map[string]string{"foo": "bar"} Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully update the provided object", func() { + It("should successfully update the provided object", func(ctx SpecContext) { By("updating the Deployment") err = getClient().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) @@ -208,7 +290,7 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should successfully update the provided object when namespace is not provided", func() { + It("should successfully update the provided object when namespace is not provided", func(ctx SpecContext) { By("updating the Deployment") dep.SetNamespace("") err = getClient().Update(ctx, dep) @@ -222,14 +304,14 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should not update when object namespace is different", func() { + It("should not update when object namespace is different", func(ctx SpecContext) { By("updating the Deployment") dep.SetNamespace("non-default") err = getClient().Update(ctx, dep) Expect(err).To(HaveOccurred()) }) - It("should not update any object from other namespace", func() { + It("should not update any object from other namespace", func(ctx SpecContext) { By("creating a new namespace") tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "non-default-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) @@ -266,7 +348,7 @@ var _ = Describe("NamespacedClient", func() { deleteNamespace(ctx, tns) }) - It("should update a cluster scoped resource", func() { + It("should update a cluster scoped resource", func(ctx SpecContext) { changedCR := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("clusterRole-%v", count), @@ -302,16 +384,16 @@ var _ = Describe("NamespacedClient", func() { Describe("Patch", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully modify the object using Patch", func() { + It("should successfully modify the object using Patch", func(ctx SpecContext) { By("Applying Patch") err = getClient().Patch(ctx, dep, client.RawPatch(types.MergePatchType, generatePatch())) Expect(err).NotTo(HaveOccurred()) @@ -323,7 +405,7 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.GetNamespace()).To(Equal(ns)) }) - It("should successfully modify the object using Patch when namespace is not provided", func() { + It("should successfully modify the object using Patch when namespace is not provided", func(ctx SpecContext) { By("Applying Patch") dep.SetNamespace("") err = getClient().Patch(ctx, dep, client.RawPatch(types.MergePatchType, generatePatch())) @@ -336,13 +418,13 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.GetNamespace()).To(Equal(ns)) }) - It("should not modify the object when namespace of the object is different", func() { + It("should not modify the object when namespace of the object is different", func(ctx SpecContext) { dep.SetNamespace("non-default") err = getClient().Patch(ctx, dep, client.RawPatch(types.MergePatchType, generatePatch())) Expect(err).To(HaveOccurred()) }) - It("should not modify an object from a different namespace", func() { + It("should not modify an object from a different namespace", func(ctx SpecContext) { By("creating a new namespace") tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "non-default-2"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) @@ -377,7 +459,7 @@ var _ = Describe("NamespacedClient", func() { deleteNamespace(ctx, tns) }) - It("should successfully modify cluster scoped resource", func() { + It("should successfully modify cluster scoped resource", func(ctx SpecContext) { cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("clusterRole-%v", count), @@ -411,15 +493,15 @@ var _ = Describe("NamespacedClient", func() { Describe("Delete and DeleteAllOf", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully delete an object when namespace is not specified", func() { + It("should successfully delete an object when namespace is not specified", func(ctx SpecContext) { By("deleting the object") dep.SetNamespace("") err = getClient().Delete(ctx, dep) @@ -430,7 +512,7 @@ var _ = Describe("NamespacedClient", func() { Expect(err).To(HaveOccurred()) }) - It("should successfully delete all of the deployments in the given namespace", func() { + It("should successfully delete all of the deployments in the given namespace", func(ctx SpecContext) { By("Deleting all objects in the namespace") err = getClient().DeleteAllOf(ctx, dep) Expect(err).NotTo(HaveOccurred()) @@ -440,7 +522,7 @@ var _ = Describe("NamespacedClient", func() { Expect(err).To(HaveOccurred()) }) - It("should not delete deployments in other namespaces", func() { + It("should not delete deployments in other namespaces", func(ctx SpecContext) { tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "non-default-3"}} _, err = clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -480,22 +562,22 @@ var _ = Describe("NamespacedClient", func() { }) }) - Describe("StatusWriter", func() { + Describe("SubResourceWriter", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should change objects via update status", func() { + It("should change objects via update status", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - Expect(getClient().Status().Update(ctx, changedDep)).NotTo(HaveOccurred()) + Expect(getClient().SubResource("status").Update(ctx, changedDep)).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -504,19 +586,19 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Status.Replicas).To(BeEquivalentTo(99)) }) - It("should not change objects via update status when object namespace is different", func() { + It("should not change objects via update status when object namespace is different", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.SetNamespace("test") changedDep.Status.Replicas = 99 - Expect(getClient().Status().Update(ctx, changedDep)).To(HaveOccurred()) + Expect(getClient().SubResource("status").Update(ctx, changedDep)).To(HaveOccurred()) }) - It("should change objects via status patch", func() { + It("should change objects via status patch", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - Expect(getClient().Status().Patch(ctx, changedDep, client.MergeFrom(dep))).NotTo(HaveOccurred()) + Expect(getClient().SubResource("status").Patch(ctx, changedDep, client.MergeFrom(dep))).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -525,17 +607,59 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Status.Replicas).To(BeEquivalentTo(99)) }) - It("should not change objects via status patch when object namespace is different", func() { + It("should not change objects via status patch when object namespace is different", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 changedDep.SetNamespace("test") - Expect(getClient().Status().Patch(ctx, changedDep, client.MergeFrom(dep))).To(HaveOccurred()) + Expect(getClient().SubResource("status").Patch(ctx, changedDep, client.MergeFrom(dep))).To(HaveOccurred()) + }) + + It("should change objects via status apply", func(ctx SpecContext) { + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "test-owner") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(99)), + }) + + Expect(getClient().SubResource("status").Apply(ctx, deploymentAC, client.FieldOwner("test-owner"))).To(Succeed()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual.GetNamespace()).To(BeEquivalentTo(ns)) + Expect(actual.Status.Replicas).To(BeEquivalentTo(99)) + }) + + It("should set namespace on ApplyConfiguration when applying via SubResource", func(ctx SpecContext) { + deploymentAC := appsv1applyconfigurations.Deployment(dep.Name, "") + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(50)), + }) + + Expect(getClient().SubResource("status").Apply(ctx, deploymentAC, client.FieldOwner("test-owner"))).To(Succeed()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual.GetNamespace()).To(BeEquivalentTo(ns)) + Expect(actual.Status.Replicas).To(BeEquivalentTo(50)) + }) + + It("should fail when applying via SubResource with conflicting namespace", func(ctx SpecContext) { + deploymentAC := appsv1applyconfigurations.Deployment(dep.Name, "different-namespace") + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(25)), + }) + + err := getClient().SubResource("status").Apply(ctx, deploymentAC, client.FieldOwner("test-owner")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("namespace")) }) }) Describe("Test on invalid objects", func() { - It("should refuse to perform operations on invalid object", func() { + It("should refuse to perform operations on invalid object", func(ctx SpecContext) { err := getClient().Create(ctx, nil) Expect(err).To(HaveOccurred()) diff --git a/pkg/client/options.go b/pkg/client/options.go index aa2299eac0..a6b921171a 100644 --- a/pkg/client/options.go +++ b/pkg/client/options.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "k8s.io/utils/ptr" ) // {{{ "Functional" Option Interfaces @@ -37,6 +38,12 @@ type DeleteOption interface { ApplyToDelete(*DeleteOptions) } +// GetOption is some configuration that modifies options for a get request. +type GetOption interface { + // ApplyToGet applies this configuration to the given get options. + ApplyToGet(*GetOptions) +} + // ListOption is some configuration that modifies options for a list request. type ListOption interface { // ApplyToList applies this configuration to the given list options. @@ -55,12 +62,47 @@ type PatchOption interface { ApplyToPatch(*PatchOptions) } +// ApplyOption is some configuration that modifies options for an apply request. +type ApplyOption interface { + // ApplyToApply applies this configuration to the given apply options. + ApplyToApply(*ApplyOptions) +} + // DeleteAllOfOption is some configuration that modifies options for a delete request. type DeleteAllOfOption interface { // ApplyToDeleteAllOf applies this configuration to the given deletecollection options. ApplyToDeleteAllOf(*DeleteAllOfOptions) } +// SubResourceGetOption modifies options for a SubResource Get request. +type SubResourceGetOption interface { + ApplyToSubResourceGet(*SubResourceGetOptions) +} + +// SubResourceUpdateOption is some configuration that modifies options for a update request. +type SubResourceUpdateOption interface { + // ApplyToSubResourceUpdate applies this configuration to the given update options. + ApplyToSubResourceUpdate(*SubResourceUpdateOptions) +} + +// SubResourceCreateOption is some configuration that modifies options for a create request. +type SubResourceCreateOption interface { + // ApplyToSubResourceCreate applies this configuration to the given create options. + ApplyToSubResourceCreate(*SubResourceCreateOptions) +} + +// SubResourcePatchOption configures a subresource patch request. +type SubResourcePatchOption interface { + // ApplyToSubResourcePatch applies the configuration on the given patch options. + ApplyToSubResourcePatch(*SubResourcePatchOptions) +} + +// SubResourceApplyOption configures a subresource apply request. +type SubResourceApplyOption interface { + // ApplyToSubResourceApply applies the configuration on the given patch options. + ApplyToSubResourceApply(*SubResourceApplyOptions) +} + // }}} // {{{ Multi-Type Options @@ -86,14 +128,36 @@ func (dryRunAll) ApplyToPatch(opts *PatchOptions) { opts.DryRun = []string{metav1.DryRunAll} } -// ApplyToPatch applies this configuration to the given delete options. +// ApplyToApply applies this configuration to the given apply options. +func (dryRunAll) ApplyToApply(opts *ApplyOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + +// ApplyToDelete applies this configuration to the given delete options. func (dryRunAll) ApplyToDelete(opts *DeleteOptions) { opts.DryRun = []string{metav1.DryRunAll} } + func (dryRunAll) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) { opts.DryRun = []string{metav1.DryRunAll} } +func (dryRunAll) ApplyToSubResourceCreate(opts *SubResourceCreateOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + +func (dryRunAll) ApplyToSubResourceUpdate(opts *SubResourceUpdateOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + +func (dryRunAll) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + +func (dryRunAll) ApplyToSubResourceApply(opts *SubResourceApplyOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + // FieldOwner set the field manager name for the given server-side apply patch. type FieldOwner string @@ -112,6 +176,64 @@ func (f FieldOwner) ApplyToUpdate(opts *UpdateOptions) { opts.FieldManager = string(f) } +// ApplyToApply applies this configuration to the given apply options. +func (f FieldOwner) ApplyToApply(opts *ApplyOptions) { + opts.FieldManager = string(f) +} + +// ApplyToSubResourcePatch applies this configuration to the given patch options. +func (f FieldOwner) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { + opts.FieldManager = string(f) +} + +// ApplyToSubResourceCreate applies this configuration to the given create options. +func (f FieldOwner) ApplyToSubResourceCreate(opts *SubResourceCreateOptions) { + opts.FieldManager = string(f) +} + +// ApplyToSubResourceUpdate applies this configuration to the given update options. +func (f FieldOwner) ApplyToSubResourceUpdate(opts *SubResourceUpdateOptions) { + opts.FieldManager = string(f) +} + +// ApplyToSubResourceApply applies this configuration to the given apply options. +func (f FieldOwner) ApplyToSubResourceApply(opts *SubResourceApplyOptions) { + opts.FieldManager = string(f) +} + +// FieldValidation configures field validation for the given requests. +type FieldValidation string + +// ApplyToPatch applies this configuration to the given patch options. +func (f FieldValidation) ApplyToPatch(opts *PatchOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToCreate applies this configuration to the given create options. +func (f FieldValidation) ApplyToCreate(opts *CreateOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToUpdate applies this configuration to the given update options. +func (f FieldValidation) ApplyToUpdate(opts *UpdateOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToSubResourcePatch applies this configuration to the given patch options. +func (f FieldValidation) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToSubResourceCreate applies this configuration to the given create options. +func (f FieldValidation) ApplyToSubResourceCreate(opts *SubResourceCreateOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToSubResourceUpdate applies this configuration to the given update options. +func (f FieldValidation) ApplyToSubResourceUpdate(opts *SubResourceUpdateOptions) { + opts.FieldValidation = string(f) +} + // }}} // {{{ Create Options @@ -130,6 +252,24 @@ type CreateOptions struct { // this request. It must be set with server-side apply. FieldManager string + // fieldValidation instructs the server on how to handle + // objects in the request (POST/PUT/PATCH) containing unknown + // or duplicate fields. Valid values are: + // - Ignore: This will ignore any unknown fields that are silently + // dropped from the object, and will ignore all but the last duplicate + // field that the decoder encounters. This is the default behavior + // prior to v1.23. + // - Warn: This will send a warning via the standard warning response + // header for each unknown field that is dropped from the object, and + // for each duplicate field that is encountered. The request will + // still succeed if there are no other errors, and will only persist + // the last of any duplicate fields. This is the default in v1.23+ + // - Strict: This will fail the request with a BadRequest error if + // any unknown fields would be dropped from the object, or if any + // duplicate fields are present. The error returned from the server + // will contain all unknown and duplicate fields encountered. + FieldValidation string + // Raw represents raw CreateOptions, as passed to the API server. Raw *metav1.CreateOptions } @@ -146,6 +286,7 @@ func (o *CreateOptions) AsCreateOptions() *metav1.CreateOptions { o.Raw.DryRun = o.DryRun o.Raw.FieldManager = o.FieldManager + o.Raw.FieldValidation = o.FieldValidation return o.Raw } @@ -166,6 +307,9 @@ func (o *CreateOptions) ApplyToCreate(co *CreateOptions) { if o.FieldManager != "" { co.FieldManager = o.FieldManager } + if o.FieldValidation != "" { + co.FieldValidation = o.FieldValidation + } if o.Raw != nil { co.Raw = o.Raw } @@ -311,6 +455,54 @@ func (p PropagationPolicy) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) { // }}} +// {{{ Get Options + +// GetOptions contains options for get operation. +// Now it only has a Raw field, with support for specific resourceVersion. +type GetOptions struct { + // Raw represents raw GetOptions, as passed to the API server. Note + // that these may not be respected by all implementations of interface. + Raw *metav1.GetOptions + + // UnsafeDisableDeepCopy indicates not to deep copy objects during get object. + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + // +optional + UnsafeDisableDeepCopy *bool +} + +var _ GetOption = &GetOptions{} + +// ApplyToGet implements GetOption for GetOptions. +func (o *GetOptions) ApplyToGet(lo *GetOptions) { + if o.Raw != nil { + lo.Raw = o.Raw + } + if o.UnsafeDisableDeepCopy != nil { + lo.UnsafeDisableDeepCopy = o.UnsafeDisableDeepCopy + } +} + +// AsGetOptions returns these options as a flattened metav1.GetOptions. +// This may mutate the Raw field. +func (o *GetOptions) AsGetOptions() *metav1.GetOptions { + if o == nil || o.Raw == nil { + return &metav1.GetOptions{} + } + return o.Raw +} + +// ApplyOptions applies the given get options on these options, +// and then returns itself (for convenient chaining). +func (o *GetOptions) ApplyOptions(opts []GetOption) *GetOptions { + for _, opt := range opts { + opt.ApplyToGet(o) + } + return o +} + +// }}} + // {{{ List Options // ListOptions contains options for limiting or filtering results. @@ -318,12 +510,12 @@ func (p PropagationPolicy) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) { // pre-parsed selectors (since generally, selectors will be executed // against the cache). type ListOptions struct { - // LabelSelector filters results by label. Use SetLabelSelector to + // LabelSelector filters results by label. Use labels.Parse() to // set from raw string form. LabelSelector labels.Selector // FieldSelector filters results by a particular field. In order // to use this with cache-based implementations, restrict usage to - // a single field-value pair that's been added to the indexers. + // exact match field-value pair that's been added to the indexers. FieldSelector fields.Selector // Namespace represents the namespace to list for, or empty for @@ -341,6 +533,12 @@ type ListOptions struct { // it has expired. This field is not supported if watch is true in the Raw ListOptions. Continue string + // UnsafeDisableDeepCopy indicates not to deep copy objects during list objects. + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + // +optional + UnsafeDisableDeepCopy *bool + // Raw represents raw ListOptions, as passed to the API server. Note // that these may not be respected by all implementations of interface, // and the LabelSelector, FieldSelector, Limit and Continue fields are ignored. @@ -369,6 +567,9 @@ func (o *ListOptions) ApplyToList(lo *ListOptions) { if o.Continue != "" { lo.Continue = o.Continue } + if o.UnsafeDisableDeepCopy != nil { + lo.UnsafeDisableDeepCopy = o.UnsafeDisableDeepCopy + } } // AsListOptions returns these options as a flattened metav1.ListOptions. @@ -408,8 +609,16 @@ type MatchingLabels map[string]string // ApplyToList applies this configuration to the given list options. func (m MatchingLabels) ApplyToList(opts *ListOptions) { // TODO(directxman12): can we avoid reserializing this over and over? - sel := labels.SelectorFromValidatedSet(map[string]string(m)) - opts.LabelSelector = sel + if opts.LabelSelector == nil { + opts.LabelSelector = labels.SelectorFromValidatedSet(map[string]string(m)) + return + } + // If there's already a selector, we need to AND the two together. + noValidSel := labels.SelectorFromValidatedSet(map[string]string(m)) + reqs, _ := noValidSel.Requirements() + for _, req := range reqs { + opts.LabelSelector = opts.LabelSelector.Add(req) + } } // ApplyToDeleteAllOf applies this configuration to the given an List options. @@ -423,14 +632,17 @@ type HasLabels []string // ApplyToList applies this configuration to the given list options. func (m HasLabels) ApplyToList(opts *ListOptions) { - sel := labels.NewSelector() + if opts.LabelSelector == nil { + opts.LabelSelector = labels.NewSelector() + } + // TODO: ignore invalid labels will result in an empty selector. + // This is inconsistent to the that of MatchingLabels. for _, label := range m { r, err := labels.NewRequirement(label, selection.Exists, nil) if err == nil { - sel = sel.Add(*r) + opts.LabelSelector = opts.LabelSelector.Add(*r) } } - opts.LabelSelector = sel } // ApplyToDeleteAllOf applies this configuration to the given an List options. @@ -447,6 +659,9 @@ type MatchingLabelsSelector struct { // ApplyToList applies this configuration to the given list options. func (m MatchingLabelsSelector) ApplyToList(opts *ListOptions) { + if m.Selector == nil { + m.Selector = labels.Nothing() + } opts.LabelSelector = m } @@ -480,6 +695,9 @@ type MatchingFieldsSelector struct { // ApplyToList applies this configuration to the given list options. func (m MatchingFieldsSelector) ApplyToList(opts *ListOptions) { + if m.Selector == nil { + m.Selector = fields.Nothing() + } opts.FieldSelector = m } @@ -501,6 +719,11 @@ func (n InNamespace) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) { n.ApplyToList(&opts.ListOptions) } +// AsSelector returns a selector that matches objects in the given namespace. +func (n InNamespace) AsSelector() fields.Selector { + return fields.SelectorFromSet(fields.Set{"metadata.namespace": string(n)}) +} + // Limit specifies the maximum number of results to return from the server. // Limit does not implement DeleteAllOfOption interface because the server // does not support setting it for deletecollection operations. @@ -511,6 +734,24 @@ func (l Limit) ApplyToList(opts *ListOptions) { opts.Limit = int64(l) } +// UnsafeDisableDeepCopyOption indicates not to deep copy objects during list objects. +// Be very careful with this, when enabled you must DeepCopy any object before mutating it, +// otherwise you will mutate the object in the cache. +type UnsafeDisableDeepCopyOption bool + +// ApplyToGet applies this configuration to the given an Get options. +func (d UnsafeDisableDeepCopyOption) ApplyToGet(opts *GetOptions) { + opts.UnsafeDisableDeepCopy = ptr.To(bool(d)) +} + +// ApplyToList applies this configuration to the given an List options. +func (d UnsafeDisableDeepCopyOption) ApplyToList(opts *ListOptions) { + opts.UnsafeDisableDeepCopy = ptr.To(bool(d)) +} + +// UnsafeDisableDeepCopy indicates not to deep copy objects during list objects. +const UnsafeDisableDeepCopy = UnsafeDisableDeepCopyOption(true) + // Continue sets a continuation token to retrieve chunks of results when using limit. // Continue does not implement DeleteAllOfOption interface because the server // does not support setting it for deletecollection operations. @@ -539,6 +780,24 @@ type UpdateOptions struct { // this request. It must be set with server-side apply. FieldManager string + // fieldValidation instructs the server on how to handle + // objects in the request (POST/PUT/PATCH) containing unknown + // or duplicate fields. Valid values are: + // - Ignore: This will ignore any unknown fields that are silently + // dropped from the object, and will ignore all but the last duplicate + // field that the decoder encounters. This is the default behavior + // prior to v1.23. + // - Warn: This will send a warning via the standard warning response + // header for each unknown field that is dropped from the object, and + // for each duplicate field that is encountered. The request will + // still succeed if there are no other errors, and will only persist + // the last of any duplicate fields. This is the default in v1.23+ + // - Strict: This will fail the request with a BadRequest error if + // any unknown fields would be dropped from the object, or if any + // duplicate fields are present. The error returned from the server + // will contain all unknown and duplicate fields encountered. + FieldValidation string + // Raw represents raw UpdateOptions, as passed to the API server. Raw *metav1.UpdateOptions } @@ -555,6 +814,7 @@ func (o *UpdateOptions) AsUpdateOptions() *metav1.UpdateOptions { o.Raw.DryRun = o.DryRun o.Raw.FieldManager = o.FieldManager + o.Raw.FieldValidation = o.FieldValidation return o.Raw } @@ -577,6 +837,9 @@ func (o *UpdateOptions) ApplyToUpdate(uo *UpdateOptions) { if o.FieldManager != "" { uo.FieldManager = o.FieldManager } + if o.FieldValidation != "" { + uo.FieldValidation = o.FieldValidation + } if o.Raw != nil { uo.Raw = o.Raw } @@ -605,6 +868,24 @@ type PatchOptions struct { // this request. It must be set with server-side apply. FieldManager string + // fieldValidation instructs the server on how to handle + // objects in the request (POST/PUT/PATCH) containing unknown + // or duplicate fields. Valid values are: + // - Ignore: This will ignore any unknown fields that are silently + // dropped from the object, and will ignore all but the last duplicate + // field that the decoder encounters. This is the default behavior + // prior to v1.23. + // - Warn: This will send a warning via the standard warning response + // header for each unknown field that is dropped from the object, and + // for each duplicate field that is encountered. The request will + // still succeed if there are no other errors, and will only persist + // the last of any duplicate fields. This is the default in v1.23+ + // - Strict: This will fail the request with a BadRequest error if + // any unknown fields would be dropped from the object, or if any + // duplicate fields are present. The error returned from the server + // will contain all unknown and duplicate fields encountered. + FieldValidation string + // Raw represents raw PatchOptions, as passed to the API server. Raw *metav1.PatchOptions } @@ -628,9 +909,18 @@ func (o *PatchOptions) AsPatchOptions() *metav1.PatchOptions { o.Raw = &metav1.PatchOptions{} } - o.Raw.DryRun = o.DryRun - o.Raw.Force = o.Force - o.Raw.FieldManager = o.FieldManager + if o.DryRun != nil { + o.Raw.DryRun = o.DryRun + } + if o.Force != nil { + o.Raw.Force = o.Force + } + if o.FieldManager != "" { + o.Raw.FieldManager = o.FieldManager + } + if o.FieldValidation != "" { + o.Raw.FieldValidation = o.FieldValidation + } return o.Raw } @@ -647,6 +937,9 @@ func (o *PatchOptions) ApplyToPatch(po *PatchOptions) { if o.FieldManager != "" { po.FieldManager = o.FieldManager } + if o.FieldValidation != "" { + po.FieldValidation = o.FieldValidation + } if o.Raw != nil { po.Raw = o.Raw } @@ -660,8 +953,19 @@ var ForceOwnership = forceOwnership{} type forceOwnership struct{} func (forceOwnership) ApplyToPatch(opts *PatchOptions) { - definitelyTrue := true - opts.Force = &definitelyTrue + opts.Force = ptr.To(true) +} + +func (forceOwnership) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { + opts.Force = ptr.To(true) +} + +func (forceOwnership) ApplyToApply(opts *ApplyOptions) { + opts.Force = ptr.To(true) +} + +func (forceOwnership) ApplyToSubResourceApply(opts *SubResourceApplyOptions) { + opts.Force = ptr.To(true) } // }}} @@ -695,3 +999,57 @@ func (o *DeleteAllOfOptions) ApplyToDeleteAllOf(do *DeleteAllOfOptions) { } // }}} + +// ApplyOptions are the options for an apply request. +type ApplyOptions struct { + // When present, indicates that modifications should not be + // persisted. An invalid or unrecognized dryRun directive will + // result in an error response and no further processing of the + // request. Valid values are: + // - All: all dry run stages will be processed + DryRun []string + + // Force is going to "force" Apply requests. It means user will + // re-acquire conflicting fields owned by other people. + Force *bool + + // fieldManager is a name associated with the actor or entity + // that is making these changes. The value must be less than or + // 128 characters long, and only contain printable characters, + // as defined by https://golang.org/pkg/unicode/#IsPrint. This + // field is required. + // + // +required + FieldManager string +} + +// ApplyOptions applies the given opts onto the ApplyOptions +func (o *ApplyOptions) ApplyOptions(opts []ApplyOption) *ApplyOptions { + for _, opt := range opts { + opt.ApplyToApply(o) + } + return o +} + +// ApplyToApply applies the given opts onto the ApplyOptions +func (o *ApplyOptions) ApplyToApply(opts *ApplyOptions) { + if o.DryRun != nil { + opts.DryRun = o.DryRun + } + if o.Force != nil { + opts.Force = o.Force + } + + if o.FieldManager != "" { + opts.FieldManager = o.FieldManager + } +} + +// AsPatchOptions constructs patch options from the given ApplyOptions +func (o *ApplyOptions) AsPatchOptions() *metav1.PatchOptions { + return &metav1.PatchOptions{ + DryRun: o.DryRun, + Force: o.Force, + FieldManager: o.FieldManager, + } +} diff --git a/pkg/client/options_test.go b/pkg/client/options_test.go index 36447a3f3c..082586bca3 100644 --- a/pkg/client/options_test.go +++ b/pkg/client/options_test.go @@ -17,13 +17,13 @@ limitations under the License. package client_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" - utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -36,12 +36,38 @@ var _ = Describe("ListOptions", func() { o.ApplyToList(newListOpts) Expect(newListOpts).To(Equal(o)) }) + It("Should set LabelSelector with MatchingLabelsSelector", func() { + labelSelector, err := labels.Parse("a=b") + Expect(err).NotTo(HaveOccurred()) + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingLabelsSelector{Selector: labelSelector}}) + expectedListOpts := &client.ListOptions{LabelSelector: client.MatchingLabelsSelector{Selector: labelSelector}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) + It("Should set LabelSelector to nothing with empty MatchingLabelsSelector", func() { + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingLabelsSelector{}}) + expectedListOpts := &client.ListOptions{LabelSelector: client.MatchingLabelsSelector{Selector: labels.Nothing()}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) It("Should set FieldSelector", func() { o := &client.ListOptions{FieldSelector: fields.Nothing()} newListOpts := &client.ListOptions{} o.ApplyToList(newListOpts) Expect(newListOpts).To(Equal(o)) }) + It("Should set FieldSelector with MatchingFieldsSelector", func() { + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingFieldsSelector{Selector: fields.Nothing()}}) + expectedListOpts := &client.ListOptions{FieldSelector: client.MatchingFieldsSelector{Selector: fields.Nothing()}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) + It("Should set FieldSelector to nothing with empty MatchingFieldsSelector", func() { + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingFieldsSelector{}}) + expectedListOpts := &client.ListOptions{FieldSelector: client.MatchingFieldsSelector{Selector: fields.Nothing()}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) It("Should set Namespace", func() { o := &client.ListOptions{Namespace: "my-ns"} newListOpts := &client.ListOptions{} @@ -66,6 +92,19 @@ var _ = Describe("ListOptions", func() { o.ApplyToList(newListOpts) Expect(newListOpts).To(Equal(o)) }) + It("Should set UnsafeDisableDeepCopy", func() { + definitelyTrue := true + o := &client.ListOptions{UnsafeDisableDeepCopy: &definitelyTrue} + newListOpts := &client.ListOptions{} + o.ApplyToList(newListOpts) + Expect(newListOpts).To(Equal(o)) + }) + It("Should set UnsafeDisableDeepCopy through option", func() { + listOpts := &client.ListOptions{} + client.UnsafeDisableDeepCopy.ApplyToList(listOpts) + Expect(listOpts.UnsafeDisableDeepCopy).ToNot(BeNil()) + Expect(*listOpts.UnsafeDisableDeepCopy).To(BeTrue()) + }) It("Should not set anything", func() { o := &client.ListOptions{} newListOpts := &client.ListOptions{} @@ -74,36 +113,79 @@ var _ = Describe("ListOptions", func() { }) }) +var _ = Describe("GetOptions", func() { + It("Should set Raw", func() { + o := &client.GetOptions{Raw: &metav1.GetOptions{ResourceVersion: "RV0"}} + newGetOpts := &client.GetOptions{} + o.ApplyToGet(newGetOpts) + Expect(newGetOpts).To(Equal(o)) + }) + It("Should set UnsafeDisableDeepCopy", func() { + definitelyTrue := true + o := &client.GetOptions{UnsafeDisableDeepCopy: &definitelyTrue} + newGetOpts := &client.GetOptions{} + o.ApplyToGet(newGetOpts) + Expect(newGetOpts).To(Equal(o)) + }) + It("Should set UnsafeDisableDeepCopy through option", func() { + getOpts := &client.GetOptions{} + client.UnsafeDisableDeepCopy.ApplyToGet(getOpts) + Expect(getOpts.UnsafeDisableDeepCopy).ToNot(BeNil()) + Expect(*getOpts.UnsafeDisableDeepCopy).To(BeTrue()) + }) +}) + +var _ = Describe("ApplyOptions", func() { + It("Should set DryRun", func() { + o := &client.ApplyOptions{DryRun: []string{"Hello", "Theodore"}} + newApplyOpts := &client.ApplyOptions{} + o.ApplyToApply(newApplyOpts) + Expect(newApplyOpts).To(Equal(o)) + }) + It("Should set Force", func() { + o := &client.ApplyOptions{Force: ptr.To(true)} + newApplyOpts := &client.ApplyOptions{} + o.ApplyToApply(newApplyOpts) + Expect(newApplyOpts).To(Equal(o)) + }) + It("Should set FieldManager", func() { + o := &client.ApplyOptions{FieldManager: "field-manager"} + newApplyOpts := &client.ApplyOptions{} + o.ApplyToApply(newApplyOpts) + Expect(newApplyOpts).To(Equal(o)) + }) +}) + var _ = Describe("CreateOptions", func() { It("Should set DryRun", func() { o := &client.CreateOptions{DryRun: []string{"Hello", "Theodore"}} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) It("Should set FieldManager", func() { o := &client.CreateOptions{FieldManager: "FieldManager"} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) It("Should set Raw", func() { o := &client.CreateOptions{Raw: &metav1.CreateOptions{DryRun: []string{"Bye", "Theodore"}}} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) It("Should not set anything", func() { o := &client.CreateOptions{} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) }) var _ = Describe("DeleteOptions", func() { It("Should set GracePeriodSeconds", func() { - o := &client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64Ptr(42)} + o := &client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(42))} newDeleteOpts := &client.DeleteOptions{} o.ApplyToDelete(newDeleteOpts) Expect(newDeleteOpts).To(Equal(o)) @@ -176,7 +258,7 @@ var _ = Describe("PatchOptions", func() { Expect(newPatchOpts).To(Equal(o)) }) It("Should set Force", func() { - o := &client.PatchOptions{Force: utilpointer.BoolPtr(true)} + o := &client.PatchOptions{Force: ptr.To(true)} newPatchOpts := &client.PatchOptions{} o.ApplyToPatch(newPatchOpts) Expect(newPatchOpts).To(Equal(o)) @@ -209,7 +291,7 @@ var _ = Describe("DeleteAllOfOptions", func() { Expect(newDeleteAllOfOpts).To(Equal(o)) }) It("Should set DeleleteOptions", func() { - o := &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64Ptr(44)}} + o := &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(44))}} newDeleteAllOfOpts := &client.DeleteAllOfOptions{} o.ApplyToDeleteAllOf(newDeleteAllOfOpts) Expect(newDeleteAllOfOpts).To(Equal(o)) @@ -224,8 +306,188 @@ var _ = Describe("MatchingLabels", func() { r, _ := listOpts.LabelSelector.Requirements() _, err := labels.NewRequirement(r[0].Key(), r[0].Operator(), r[0].Values().List()) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) expectedErrMsg := `values[0][k]: Invalid value: "axahm2EJ8Phiephe2eixohbee9eGeiyees1thuozi1xoh0GiuH3diewi8iem7Nui": must be no more than 63 characters` Expect(err.Error()).To(Equal(expectedErrMsg)) }) + + It("Should add matchingLabels to existing selector", func() { + listOpts := &client.ListOptions{} + + matchingLabels := client.MatchingLabels(map[string]string{"k": "v"}) + matchingLabels2 := client.MatchingLabels(map[string]string{"k2": "v2"}) + + matchingLabels.ApplyToList(listOpts) + Expect(listOpts.LabelSelector.String()).To(Equal("k=v")) + + matchingLabels2.ApplyToList(listOpts) + Expect(listOpts.LabelSelector.String()).To(Equal("k=v,k2=v2")) + }) +}) + +var _ = Describe("DryRunAll", func() { + It("Should apply to ApplyOptions", func() { + o := &client.ApplyOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToApply(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to CreateOptions", func() { + o := &client.CreateOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToCreate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to UpdateOptions", func() { + o := &client.UpdateOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToUpdate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to PatchOptions", func() { + o := &client.PatchOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToPatch(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to DeleteOptions", func() { + o := &client.DeleteOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToDelete(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourcePatchOptions", func() { + o := &client.SubResourcePatchOptions{PatchOptions: client.PatchOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourcePatch(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourceCreateOptions", func() { + o := &client.SubResourceCreateOptions{CreateOptions: client.CreateOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourceCreate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourceUpdateOptions", func() { + o := &client.SubResourceUpdateOptions{UpdateOptions: client.UpdateOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourceUpdate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourceApplyOptions", func() { + o := &client.SubResourceApplyOptions{ApplyOptions: client.ApplyOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourceApply(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) +}) + +var _ = Describe("FieldOwner", func() { + It("Should apply to PatchOptions", func() { + o := &client.PatchOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToPatch(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to ApplyOptions", func() { + o := &client.ApplyOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToApply(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to CreateOptions", func() { + o := &client.CreateOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToCreate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to UpdateOptions", func() { + o := &client.UpdateOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToUpdate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to SubResourcePatchOptions", func() { + o := &client.SubResourcePatchOptions{PatchOptions: client.PatchOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourcePatch(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to SubResourceCreateOptions", func() { + o := &client.SubResourceCreateOptions{CreateOptions: client.CreateOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourceCreate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to SubResourceUpdateOptions", func() { + o := &client.SubResourceUpdateOptions{UpdateOptions: client.UpdateOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourceUpdate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to SubResourceApplyOptions", func() { + o := &client.SubResourceApplyOptions{ApplyOptions: client.ApplyOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourceApply(o) + Expect(o.FieldManager).To(Equal("foo")) + }) +}) + +var _ = Describe("ForceOwnership", func() { + It("Should apply to PatchOptions", func() { + o := &client.PatchOptions{} + t := client.ForceOwnership + t.ApplyToPatch(o) + Expect(*o.Force).To(BeTrue()) + }) + It("Should apply to SubResourcePatchOptions", func() { + o := &client.SubResourcePatchOptions{PatchOptions: client.PatchOptions{}} + t := client.ForceOwnership + t.ApplyToSubResourcePatch(o) + Expect(*o.Force).To(BeTrue()) + }) + It("Should apply to ApplyOptions", func() { + o := &client.ApplyOptions{} + t := client.ForceOwnership + t.ApplyToApply(o) + Expect(*o.Force).To(BeTrue()) + }) + It("Should apply to SubResourceApplyOptions", func() { + o := &client.SubResourceApplyOptions{} + t := client.ForceOwnership + t.ApplyToSubResourceApply(o) + Expect(*o.Force).To(BeTrue()) + }) +}) + +var _ = Describe("HasLabels", func() { + It("Should produce hasLabels in given order", func() { + listOpts := &client.ListOptions{} + + hasLabels := client.HasLabels([]string{"labelApe", "labelFox"}) + hasLabels.ApplyToList(listOpts) + Expect(listOpts.LabelSelector.String()).To(Equal("labelApe,labelFox")) + }) + + It("Should add hasLabels to existing hasLabels selector", func() { + listOpts := &client.ListOptions{} + + hasLabel := client.HasLabels([]string{"labelApe"}) + hasLabel.ApplyToList(listOpts) + + hasOtherLabel := client.HasLabels([]string{"labelFox"}) + hasOtherLabel.ApplyToList(listOpts) + Expect(listOpts.LabelSelector.String()).To(Equal("labelApe,labelFox")) + }) + + It("Should add hasLabels to existing matchingLabels", func() { + listOpts := &client.ListOptions{} + + matchingLabels := client.MatchingLabels(map[string]string{"k": "v"}) + matchingLabels.ApplyToList(listOpts) + + hasLabel := client.HasLabels([]string{"labelApe"}) + hasLabel.ApplyToList(listOpts) + Expect(listOpts.LabelSelector.String()).To(Equal("k=v,labelApe")) + }) }) diff --git a/pkg/client/patch.go b/pkg/client/patch.go index 10984c5342..9bd0953fdc 100644 --- a/pkg/client/patch.go +++ b/pkg/client/patch.go @@ -19,7 +19,7 @@ package client import ( "fmt" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/strategicpatch" @@ -27,6 +27,8 @@ import ( var ( // Apply uses server-side apply to patch the given object. + // + // Deprecated: Use client.Client.Apply() and client.Client.SubResource("subrsource").Apply() instead. Apply Patch = applyPatch{} // Merge uses the raw object as a merge patch, without modifications. diff --git a/pkg/client/patch_test.go b/pkg/client/patch_test.go index 7f86d2357a..c9e105ae51 100644 --- a/pkg/client/patch_test.go +++ b/pkg/client/patch_test.go @@ -19,9 +19,13 @@ package client import ( "testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func BenchmarkMergeFrom(b *testing.B) { @@ -45,12 +49,12 @@ func BenchmarkMergeFrom(b *testing.B) { }, }, ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ + ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{}, }, }, Lifecycle: &corev1.Lifecycle{ - PreStop: &corev1.Handler{ + PreStop: &corev1.LifecycleHandler{ HTTPGet: &corev1.HTTPGetAction{}, }, }, @@ -93,3 +97,26 @@ func BenchmarkMergeFrom(b *testing.B) { } }) } + +var _ = Describe("MergeFrom", func() { + It("should successfully create a patch for two large and similar in64s", func() { + var largeInt64 int64 = 9223372036854775807 + var similarLargeInt64 int64 = 9223372036854775800 + j := batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: batchv1.JobSpec{ + ActiveDeadlineSeconds: &largeInt64, + }, + } + patch := MergeFrom(j.DeepCopy()) + + j.Spec.ActiveDeadlineSeconds = &similarLargeInt64 + + data, err := patch.Data(&j) + Expect(err).NotTo(HaveOccurred()) + Expect(data).To(Equal([]byte(`{"spec":{"activeDeadlineSeconds":9223372036854775800}}`))) + }) +}) diff --git a/pkg/client/split.go b/pkg/client/split.go deleted file mode 100644 index bf4b861f39..0000000000 --- a/pkg/client/split.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package client - -import ( - "context" - "strings" - - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" -) - -// NewDelegatingClientInput encapsulates the input parameters to create a new delegating client. -type NewDelegatingClientInput struct { - CacheReader Reader - Client Client - UncachedObjects []Object - CacheUnstructured bool -} - -// NewDelegatingClient creates a new delegating client. -// -// A delegating client forms a Client by composing separate reader, writer and -// statusclient interfaces. This way, you can have an Client that reads from a -// cache and writes to the API server. -func NewDelegatingClient(in NewDelegatingClientInput) (Client, error) { - uncachedGVKs := map[schema.GroupVersionKind]struct{}{} - for _, obj := range in.UncachedObjects { - gvk, err := apiutil.GVKForObject(obj, in.Client.Scheme()) - if err != nil { - return nil, err - } - uncachedGVKs[gvk] = struct{}{} - } - - return &delegatingClient{ - scheme: in.Client.Scheme(), - mapper: in.Client.RESTMapper(), - Reader: &delegatingReader{ - CacheReader: in.CacheReader, - ClientReader: in.Client, - scheme: in.Client.Scheme(), - uncachedGVKs: uncachedGVKs, - cacheUnstructured: in.CacheUnstructured, - }, - Writer: in.Client, - StatusClient: in.Client, - }, nil -} - -type delegatingClient struct { - Reader - Writer - StatusClient - - scheme *runtime.Scheme - mapper meta.RESTMapper -} - -// Scheme returns the scheme this client is using. -func (d *delegatingClient) Scheme() *runtime.Scheme { - return d.scheme -} - -// RESTMapper returns the rest mapper this client is using. -func (d *delegatingClient) RESTMapper() meta.RESTMapper { - return d.mapper -} - -// delegatingReader forms a Reader that will cause Get and List requests for -// unstructured types to use the ClientReader while requests for any other type -// of object with use the CacheReader. This avoids accidentally caching the -// entire cluster in the common case of loading arbitrary unstructured objects -// (e.g. from OwnerReferences). -type delegatingReader struct { - CacheReader Reader - ClientReader Reader - - uncachedGVKs map[schema.GroupVersionKind]struct{} - scheme *runtime.Scheme - cacheUnstructured bool -} - -func (d *delegatingReader) shouldBypassCache(obj runtime.Object) (bool, error) { - gvk, err := apiutil.GVKForObject(obj, d.scheme) - if err != nil { - return false, err - } - // TODO: this is producing unsafe guesses that don't actually work, - // but it matches ~99% of the cases out there. - if meta.IsListType(obj) { - gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") - } - if _, isUncached := d.uncachedGVKs[gvk]; isUncached { - return true, nil - } - if !d.cacheUnstructured { - _, isUnstructured := obj.(*unstructured.Unstructured) - _, isUnstructuredList := obj.(*unstructured.UnstructuredList) - return isUnstructured || isUnstructuredList, nil - } - return false, nil -} - -// Get retrieves an obj for a given object key from the Kubernetes Cluster. -func (d *delegatingReader) Get(ctx context.Context, key ObjectKey, obj Object) error { - if isUncached, err := d.shouldBypassCache(obj); err != nil { - return err - } else if isUncached { - return d.ClientReader.Get(ctx, key, obj) - } - return d.CacheReader.Get(ctx, key, obj) -} - -// List retrieves list of objects for a given namespace and list options. -func (d *delegatingReader) List(ctx context.Context, list ObjectList, opts ...ListOption) error { - if isUncached, err := d.shouldBypassCache(list); err != nil { - return err - } else if isUncached { - return d.ClientReader.List(ctx, list, opts...) - } - return d.CacheReader.List(ctx, list, opts...) -} diff --git a/pkg/client/typed_client.go b/pkg/client/typed_client.go index dde7b21f25..66ae2e4a5c 100644 --- a/pkg/client/typed_client.go +++ b/pkg/client/typed_client.go @@ -18,32 +18,32 @@ package client import ( "context" + "fmt" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/apply" ) var _ Reader = &typedClient{} var _ Writer = &typedClient{} -var _ StatusWriter = &typedClient{} -// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes -// new clients at the time they are used, and caches the client. type typedClient struct { - cache *clientCache + resources *clientRestResources paramCodec runtime.ParameterCodec } // Create implements client.Client. func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } createOpts := &CreateOptions{} createOpts.ApplyOptions(opts) + return o.Post(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). Body(obj). VersionedParams(createOpts.AsCreateOptions(), c.paramCodec). @@ -53,17 +53,18 @@ func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOpti // Update implements client.Client. func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } updateOpts := &UpdateOptions{} updateOpts.ApplyOptions(opts) + return o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(obj). VersionedParams(updateOpts.AsUpdateOptions(), c.paramCodec). Do(ctx). @@ -72,7 +73,7 @@ func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOpti // Delete implements client.Client. func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -81,9 +82,9 @@ func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOpti deleteOpts.ApplyOptions(opts) return o.Delete(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(deleteOpts.AsDeleteOptions()). Do(ctx). Error() @@ -91,7 +92,7 @@ func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOpti // DeleteAllOf implements client.Client. func (c *typedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -110,7 +111,7 @@ func (c *typedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...Delet // Patch implements client.Client. func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -121,36 +122,68 @@ func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts . } patchOpts := &PatchOptions{} + patchOpts.ApplyOptions(opts) + return o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), c.paramCodec). + Name(o.name). + VersionedParams(patchOpts.AsPatchOptions(), c.paramCodec). Body(data). Do(ctx). Into(obj) } +func (c *typedClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + req, err := apply.NewRequest(o, obj) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + applyOpts := &ApplyOptions{} + applyOpts.ApplyOptions(opts) + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + VersionedParams(applyOpts.AsPatchOptions(), c.paramCodec). + Do(ctx). + // This is hacky, it is required because `Into` takes a `runtime.Object` and + // that is not implemented by the ApplyConfigurations. The generated clients + // don't have this problem because they deserialize into the api type, not the + // apply configuration: https://github.com/kubernetes/kubernetes/blob/22f5e01a37c0bc6a5f494dec14dd4e3688ee1d55/staging/src/k8s.io/client-go/gentype/type.go#L296-L317 + Into(runtimeObjectFromApplyConfiguration(obj)) +} + // Get implements client.Client. -func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - r, err := c.cache.getResource(obj) +func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + r, err := c.resources.getResource(obj) if err != nil { return err } + getOpts := GetOptions{} + getOpts.ApplyOptions(opts) return r.Get(). NamespaceIfScoped(key.Namespace, r.isNamespaced()). Resource(r.resource()). + VersionedParams(getOpts.AsGetOptions(), c.paramCodec). Name(key.Name).Do(ctx).Into(obj) } // List implements client.Client. func (c *typedClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error { - r, err := c.cache.getResource(obj) + r, err := c.resources.getResource(obj) if err != nil { return err } + listOpts := ListOptions{} listOpts.ApplyOptions(opts) + return r.Get(). NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()). Resource(r.resource()). @@ -159,9 +192,56 @@ func (c *typedClient) List(ctx context.Context, obj ObjectList, opts ...ListOpti Into(obj) } -// UpdateStatus used by StatusWriter to write status. -func (c *typedClient) UpdateStatus(ctx context.Context, obj Object, opts ...UpdateOption) error { - o, err := c.cache.getObjMeta(obj) +func (c *typedClient) GetSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceGetOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + getOpts := &SubResourceGetOptions{} + getOpts.ApplyOptions(opts) + + return o.Get(). + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + VersionedParams(getOpts.AsGetOptions(), c.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +func (c *typedClient) CreateSubResource(ctx context.Context, obj Object, subResourceObj Object, subResource string, opts ...SubResourceCreateOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + createOpts := &SubResourceCreateOptions{} + createOpts.ApplyOptions(opts) + + return o.Post(). + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + Body(subResourceObj). + VersionedParams(createOpts.AsCreateOptions(), c.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +// UpdateSubResource used by SubResourceWriter to write status. +func (c *typedClient) UpdateSubResource(ctx context.Context, obj Object, subResource string, opts ...SubResourceUpdateOption) error { + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -169,37 +249,91 @@ func (c *typedClient) UpdateStatus(ctx context.Context, obj Object, opts ...Upda // wrapped to improve the UX ? // It will be nice to receive an error saying the object doesn't implement // status subresource and check CRD definition + updateOpts := &SubResourceUpdateOptions{} + updateOpts.ApplyOptions(opts) + + body := obj + if updateOpts.SubResourceBody != nil { + body = updateOpts.SubResourceBody + } + if body.GetName() == "" { + body.SetName(obj.GetName()) + } + if body.GetNamespace() == "" { + body.SetNamespace(obj.GetNamespace()) + } + return o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). - SubResource("status"). - Body(obj). - VersionedParams((&UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions(), c.paramCodec). + Name(o.name). + SubResource(subResource). + Body(body). + VersionedParams(updateOpts.AsUpdateOptions(), c.paramCodec). Do(ctx). - Into(obj) + Into(body) } -// PatchStatus used by StatusWriter to write status. -func (c *typedClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - o, err := c.cache.getObjMeta(obj) +// PatchSubResource used by SubResourceWriter to write subresource. +func (c *typedClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error { + o, err := c.resources.getObjMeta(obj) if err != nil { return err } - data, err := patch.Data(obj) + patchOpts := &SubResourcePatchOptions{} + patchOpts.ApplyOptions(opts) + + body := obj + if patchOpts.SubResourceBody != nil { + body = patchOpts.SubResourceBody + } + + data, err := patch.Data(body) if err != nil { return err } - patchOpts := &PatchOptions{} return o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). - SubResource("status"). + Name(o.name). + SubResource(subResource). Body(data). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), c.paramCodec). + VersionedParams(patchOpts.AsPatchOptions(), c.paramCodec). Do(ctx). - Into(obj) + Into(body) +} + +func (c *typedClient) ApplySubResource(ctx context.Context, obj runtime.ApplyConfiguration, subResource string, opts ...SubResourceApplyOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + + applyOpts := &SubResourceApplyOptions{} + applyOpts.ApplyOpts(opts) + + body := obj + if applyOpts.SubResourceBody != nil { + body = applyOpts.SubResourceBody + } + + req, err := apply.NewRequest(o, body) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + VersionedParams(applyOpts.AsPatchOptions(), c.paramCodec). + Do(ctx). + // This is hacky, it is required because `Into` takes a `runtime.Object` and + // that is not implemented by the ApplyConfigurations. The generated clients + // don't have this problem because they deserialize into the api type, not the + // apply configuration: https://github.com/kubernetes/kubernetes/blob/22f5e01a37c0bc6a5f494dec14dd4e3688ee1d55/staging/src/k8s.io/client-go/gentype/type.go#L296-L317 + Into(runtimeObjectFromApplyConfiguration(obj)) } diff --git a/pkg/client/unstructured_client.go b/pkg/client/unstructured_client.go index dcf15be275..d2ea6d7a32 100644 --- a/pkg/client/unstructured_client.go +++ b/pkg/client/unstructured_client.go @@ -21,96 +21,95 @@ import ( "fmt" "strings" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/apply" ) var _ Reader = &unstructuredClient{} var _ Writer = &unstructuredClient{} -var _ StatusWriter = &unstructuredClient{} -// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes -// new clients at the time they are used, and caches the client. type unstructuredClient struct { - cache *clientCache + resources *clientRestResources paramCodec runtime.ParameterCodec } // Create implements client.Client. func (uc *unstructuredClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { - u, ok := obj.(*unstructured.Unstructured) + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } createOpts := &CreateOptions{} createOpts.ApplyOptions(opts) + result := o.Post(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). Body(obj). VersionedParams(createOpts.AsCreateOptions(), uc.paramCodec). Do(ctx). Into(obj) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } // Update implements client.Client. func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - u, ok := obj.(*unstructured.Unstructured) + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } updateOpts := UpdateOptions{} updateOpts.ApplyOptions(opts) + result := o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(obj). VersionedParams(updateOpts.AsUpdateOptions(), uc.paramCodec). Do(ctx). Into(obj) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } // Delete implements client.Client. func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { + if _, ok := obj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } deleteOpts := DeleteOptions{} deleteOpts.ApplyOptions(opts) + return o.Delete(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(deleteOpts.AsDeleteOptions()). Do(ctx). Error() @@ -118,18 +117,18 @@ func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...De // DeleteAllOf implements client.Client. func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { + if _, ok := obj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } deleteAllOfOpts := DeleteAllOfOptions{} deleteAllOfOpts.ApplyOptions(opts) + return o.Delete(). NamespaceIfScoped(deleteAllOfOpts.ListOptions.Namespace, o.isNamespaced()). Resource(o.resource()). @@ -141,12 +140,11 @@ func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts // Patch implements client.Client. func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { + if _, ok := obj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } @@ -157,26 +155,57 @@ func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch } patchOpts := &PatchOptions{} + patchOpts.ApplyOptions(opts) + return o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), uc.paramCodec). + Name(o.name). + VersionedParams(patchOpts.AsPatchOptions(), uc.paramCodec). Body(data). Do(ctx). Into(obj) } +func (uc *unstructuredClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + unstructuredApplyConfig, ok := obj.(*unstructuredApplyConfiguration) + if !ok { + return fmt.Errorf("bug: unstructured client got an applyconfiguration that was not %T but %T", &unstructuredApplyConfiguration{}, obj) + } + o, err := uc.resources.getObjMeta(unstructuredApplyConfig.Unstructured) + if err != nil { + return err + } + + req, err := apply.NewRequest(o, obj) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + applyOpts := &ApplyOptions{} + applyOpts.ApplyOptions(opts) + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + VersionedParams(applyOpts.AsPatchOptions(), uc.paramCodec). + Do(ctx). + Into(unstructuredApplyConfig.Unstructured) +} + // Get implements client.Client. -func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - u, ok := obj.(*unstructured.Unstructured) +func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() - r, err := uc.cache.getResource(obj) + getOpts := GetOptions{} + getOpts.ApplyOptions(opts) + + r, err := uc.resources.getResource(obj) if err != nil { return err } @@ -184,35 +213,34 @@ func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object result := r.Get(). NamespaceIfScoped(key.Namespace, r.isNamespaced()). Resource(r.resource()). + VersionedParams(getOpts.AsGetOptions(), uc.paramCodec). Name(key.Name). Do(ctx). Into(obj) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } // List implements client.Client. func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error { - u, ok := obj.(*unstructured.UnstructuredList) + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk := u.GetObjectKind().GroupVersionKind() + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") - listOpts := ListOptions{} - listOpts.ApplyOptions(opts) - - r, err := uc.cache.getResource(obj) + r, err := uc.resources.getResource(obj) if err != nil { return err } + listOpts := ListOptions{} + listOpts.ApplyOptions(opts) + return r.Get(). NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()). Resource(r.resource()). @@ -221,57 +249,172 @@ func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ... Into(obj) } -func (uc *unstructuredClient) UpdateStatus(ctx context.Context, obj Object, opts ...UpdateOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { +func (uc *unstructuredClient) GetSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceGetOption) error { + if _, ok := obj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + if _, ok := subResourceObj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", subResourceObj) + } + + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } + getOpts := &SubResourceGetOptions{} + getOpts.ApplyOptions(opts) + + return o.Get(). + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + VersionedParams(getOpts.AsGetOptions(), uc.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +func (uc *unstructuredClient) CreateSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceCreateOption) error { + if _, ok := obj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", obj) + } + + if _, ok := subResourceObj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", subResourceObj) + } + + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + o, err := uc.resources.getObjMeta(obj) + if err != nil { + return err + } + + createOpts := &SubResourceCreateOptions{} + createOpts.ApplyOptions(opts) + + return o.Post(). + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + Body(subResourceObj). + VersionedParams(createOpts.AsCreateOptions(), uc.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +func (uc *unstructuredClient) UpdateSubResource(ctx context.Context, obj Object, subResource string, opts ...SubResourceUpdateOption) error { + if _, ok := obj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", obj) + } + + o, err := uc.resources.getObjMeta(obj) + if err != nil { + return err + } + + updateOpts := SubResourceUpdateOptions{} + updateOpts.ApplyOptions(opts) + + body := obj + if updateOpts.SubResourceBody != nil { + body = updateOpts.SubResourceBody + } + if body.GetName() == "" { + body.SetName(obj.GetName()) + } + if body.GetNamespace() == "" { + body.SetNamespace(obj.GetNamespace()) + } + return o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). - SubResource("status"). - Body(obj). - VersionedParams((&UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions(), uc.paramCodec). + Name(o.name). + SubResource(subResource). + Body(body). + VersionedParams(updateOpts.AsUpdateOptions(), uc.paramCodec). Do(ctx). - Into(obj) + Into(body) } -func (uc *unstructuredClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - u, ok := obj.(*unstructured.Unstructured) +func (uc *unstructuredClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error { + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } - data, err := patch.Data(obj) + patchOpts := &SubResourcePatchOptions{} + patchOpts.ApplyOptions(opts) + + body := obj + if patchOpts.SubResourceBody != nil { + body = patchOpts.SubResourceBody + } + + data, err := patch.Data(body) if err != nil { return err } - patchOpts := &PatchOptions{} result := o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). - SubResource("status"). + Name(o.name). + SubResource(subResource). Body(data). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), uc.paramCodec). + VersionedParams(patchOpts.AsPatchOptions(), uc.paramCodec). Do(ctx). - Into(u) + Into(body) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } + +func (uc *unstructuredClient) ApplySubResource(ctx context.Context, obj runtime.ApplyConfiguration, subResource string, opts ...SubResourceApplyOption) error { + unstructuredApplyConfig, ok := obj.(*unstructuredApplyConfiguration) + if !ok { + return fmt.Errorf("bug: unstructured client got an applyconfiguration that was not %T but %T", &unstructuredApplyConfiguration{}, obj) + } + o, err := uc.resources.getObjMeta(unstructuredApplyConfig.Unstructured) + if err != nil { + return err + } + + applyOpts := &SubResourceApplyOptions{} + applyOpts.ApplyOpts(opts) + + body := obj + if applyOpts.SubResourceBody != nil { + body = applyOpts.SubResourceBody + } + req, err := apply.NewRequest(o, body) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + VersionedParams(applyOpts.AsPatchOptions(), uc.paramCodec). + Do(ctx). + Into(unstructuredApplyConfig.Unstructured) +} diff --git a/pkg/client/watch.go b/pkg/client/watch.go index 765ca5daa6..181b22a673 100644 --- a/pkg/client/watch.go +++ b/pkg/client/watch.go @@ -21,9 +21,8 @@ import ( "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" ) @@ -33,21 +32,16 @@ func NewWithWatch(config *rest.Config, options Options) (WithWatch, error) { if err != nil { return nil, err } - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - return nil, err - } - return &watchingClient{client: client, dynamic: dynamicClient}, nil + return &watchingClient{client: client}, nil } type watchingClient struct { *client - dynamic dynamic.Interface } func (w *watchingClient) Watch(ctx context.Context, list ObjectList, opts ...ListOption) (watch.Interface, error) { switch l := list.(type) { - case *unstructured.UnstructuredList: + case runtime.Unstructured: return w.unstructuredWatch(ctx, l, opts...) case *metav1.PartialObjectMetadataList: return w.metadataWatch(ctx, l, opts...) @@ -69,9 +63,7 @@ func (w *watchingClient) listOpts(opts ...ListOption) ListOptions { func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialObjectMetadataList, opts ...ListOption) (watch.Interface, error) { gvk := obj.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") listOpts := w.listOpts(opts...) @@ -83,27 +75,23 @@ func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialO return resInt.Watch(ctx, *listOpts.AsListOptions()) } -func (w *watchingClient) unstructuredWatch(ctx context.Context, obj *unstructured.UnstructuredList, opts ...ListOption) (watch.Interface, error) { - gvk := obj.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } - - r, err := w.client.unstructuredClient.cache.getResource(obj) +func (w *watchingClient) unstructuredWatch(ctx context.Context, obj runtime.Unstructured, opts ...ListOption) (watch.Interface, error) { + r, err := w.client.unstructuredClient.resources.getResource(obj) if err != nil { return nil, err } listOpts := w.listOpts(opts...) - if listOpts.Namespace != "" && r.isNamespaced() { - return w.dynamic.Resource(r.mapping.Resource).Namespace(listOpts.Namespace).Watch(ctx, *listOpts.AsListOptions()) - } - return w.dynamic.Resource(r.mapping.Resource).Watch(ctx, *listOpts.AsListOptions()) + return r.Get(). + NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()). + Resource(r.resource()). + VersionedParams(listOpts.AsListOptions(), w.client.unstructuredClient.paramCodec). + Watch(ctx) } func (w *watchingClient) typedWatch(ctx context.Context, obj ObjectList, opts ...ListOption) (watch.Interface, error) { - r, err := w.client.typedClient.cache.getResource(obj) + r, err := w.client.typedClient.resources.getResource(obj) if err != nil { return nil, err } diff --git a/pkg/client/watch_test.go b/pkg/client/watch_test.go index 6181596b5e..8d5b3344d3 100644 --- a/pkg/client/watch_test.go +++ b/pkg/client/watch_test.go @@ -21,7 +21,7 @@ import ( "fmt" "sync/atomic" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -38,9 +38,8 @@ var _ = Describe("ClientWithWatch", func() { var count uint64 = 0 var replicaCount int32 = 2 var ns = "kube-public" - ctx := context.TODO() - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { atomic.AddUint64(&count, 1) dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("watch-deployment-name-%v", count), Namespace: ns, Labels: map[string]string{"app": fmt.Sprintf("bar-%v", count)}}, @@ -59,20 +58,20 @@ var _ = Describe("ClientWithWatch", func() { var err error dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - }, serverSideTimeoutSeconds) + }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) - }, serverSideTimeoutSeconds) + }) Describe("NewWithWatch", func() { - It("should return a new Client", func() { + It("should return a new Client", func(ctx SpecContext) { cl, err := client.NewWithWatch(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) }) - watchSuite := func(through client.ObjectList, expectedType client.Object) { + watchSuite := func(ctx context.Context, through client.ObjectList, expectedType client.Object, checkGvk bool) { cl, err := client.NewWithWatch(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -99,26 +98,35 @@ var _ = Describe("ClientWithWatch", func() { Expect(metaObject.GetName()).To(Equal(dep.Name)) Expect(metaObject.GetUID()).To(Equal(dep.UID)) + if checkGvk { + runtimeObject := event.Object + gvk := runtimeObject.GetObjectKind().GroupVersionKind() + Expect(gvk).To(Equal(schema.GroupVersionKind{ + Group: "apps", + Kind: "Deployment", + Version: "v1", + })) + } } - It("should receive a create event when watching the typed object", func() { - watchSuite(&appsv1.DeploymentList{}, &appsv1.Deployment{}) - }, 15) + It("should receive a create event when watching the typed object", func(ctx SpecContext) { + watchSuite(ctx, &appsv1.DeploymentList{}, &appsv1.Deployment{}, false) + }) - It("should receive a create event when watching the unstructured object", func() { + It("should receive a create event when watching the unstructured object", func(ctx SpecContext) { u := &unstructured.UnstructuredList{} u.SetGroupVersionKind(schema.GroupVersionKind{ Group: "apps", Kind: "Deployment", Version: "v1", }) - watchSuite(u, &unstructured.Unstructured{}) - }, 15) + watchSuite(ctx, u, &unstructured.Unstructured{}, true) + }) - It("should receive a create event when watching the metadata object", func() { + It("should receive a create event when watching the metadata object", func(ctx SpecContext) { m := &metav1.PartialObjectMetadataList{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}} - watchSuite(m, &metav1.PartialObjectMetadata{}) - }, 15) + watchSuite(ctx, m, &metav1.PartialObjectMetadata{}, false) + }) }) }) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index dfd0fa9dd8..0603f4cde5 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -19,7 +19,7 @@ package cluster import ( "context" "errors" - "time" + "net/http" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/meta" @@ -27,24 +27,25 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" ) // Cluster provides various methods to interact with a cluster. type Cluster interface { - // SetFields will set any dependencies on an object for which the object has implemented the inject - // interface - e.g. inject.Client. - // Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10. - SetFields(interface{}) error + // GetHTTPClient returns an HTTP client that can be used to talk to the apiserver + GetHTTPClient() *http.Client // GetConfig returns an initialized Config GetConfig() *rest.Config + // GetCache returns a cache.Cache + GetCache() cache.Cache + // GetScheme returns an initialized Scheme GetScheme() *runtime.Scheme @@ -57,17 +58,14 @@ type Cluster interface { // GetFieldIndexer returns a client.FieldIndexer configured with the client GetFieldIndexer() client.FieldIndexer - // GetCache returns a cache.Cache - GetCache() cache.Cache - // GetEventRecorderFor returns a new EventRecorder for the provided name GetEventRecorderFor(name string) record.EventRecorder // GetRESTMapper returns a RESTMapper GetRESTMapper() meta.RESTMapper - // GetAPIReader returns a reader that will be configured to use the API server. - // This should be used sparingly and only when the client does not fit your + // GetAPIReader returns a reader that will be configured to use the API server directly. + // This should be used sparingly and only when the cached client does not fit your // use case. GetAPIReader() client.Reader @@ -83,44 +81,45 @@ type Options struct { Scheme *runtime.Scheme // MapperProvider provides the rest mapper used to map go types to Kubernetes APIs - MapperProvider func(c *rest.Config) (meta.RESTMapper, error) + MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) // Logger is the logger that should be used by this Cluster. // If none is set, it defaults to log.Log global logger. Logger logr.Logger - // SyncPeriod determines the minimum frequency at which watched resources are - // reconciled. A lower period will correct entropy more quickly, but reduce - // responsiveness to change if there are many watched resources. Change this - // value only if you know what you are doing. Defaults to 10 hours if unset. - // there will a 10 percent jitter between the SyncPeriod of all controllers - // so that all controllers will not send list requests simultaneously. - SyncPeriod *time.Duration + // HTTPClient is the http client that will be used to create the default + // Cache and Client. If not set the rest.HTTPClientFor function will be used + // to create the http client. + HTTPClient *http.Client - // Namespace if specified restricts the manager's cache to watch objects in - // the desired namespace Defaults to all namespaces - // - // Note: If a namespace is specified, controllers can still Watch for a - // cluster-scoped resource (e.g Node). For namespaced resources the cache - // will only hold objects from the desired namespace. - Namespace string + // Cache is the cache.Options that will be used to create the default Cache. + // By default, the cache will watch and list requested objects in all namespaces. + Cache cache.Options // NewCache is the function that will create the cache to be used // by the manager. If not set this will use the default new cache function. + // + // When using a custom NewCache, the Cache options will be passed to the + // NewCache function. + // + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewCache if you know what you are doing. NewCache cache.NewCacheFunc - // NewClient is the func that creates the client to be used by the manager. - // If not set this will create the default DelegatingClient that will - // use the cache for reads and the client for writes. - NewClient NewClientFunc - - // ClientDisableCacheFor tells the client that, if any cache is used, to bypass it - // for the given objects. - ClientDisableCacheFor []client.Object + // Client is the client.Options that will be used to create the default Client. + // By default, the client will use the cache for reads and direct calls for writes. + Client client.Options - // DryRunClient specifies whether the client should be configured to enforce - // dryRun mode. - DryRunClient bool + // NewClient is the func that creates the client to be used by the manager. + // If not set this will create a Client backed by a Cache for read operations + // and a direct Client for write operations. + // + // When using a custom NewClient, the Client options will be passed to the + // NewClient function. + // + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewClient if you know what you are doing. + NewClient client.NewClientFunc // EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API // Use this to customize the event correlator and spam filter @@ -136,7 +135,7 @@ type Options struct { makeBroadcaster intrec.EventBroadcasterProducer // Dependency injection for testing - newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) + newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) } // Option can be used to manipulate Options. @@ -148,56 +147,100 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { return nil, errors.New("must specify Config") } + originalConfig := config + + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + options := Options{} for _, opt := range opts { opt(&options) } - options = setOptionsDefaults(options) + options, err := setOptionsDefaults(options, config) + if err != nil { + options.Logger.Error(err, "Failed to set defaults") + return nil, err + } // Create the mapper provider - mapper, err := options.MapperProvider(config) + mapper, err := options.MapperProvider(config, options.HTTPClient) if err != nil { options.Logger.Error(err, "Failed to get API Group-Resources") return nil, err } // Create the cache for the cached read client and registering informers - cache, err := options.NewCache(config, cache.Options{Scheme: options.Scheme, Mapper: mapper, Resync: options.SyncPeriod, Namespace: options.Namespace}) + cacheOpts := options.Cache + { + if cacheOpts.Scheme == nil { + cacheOpts.Scheme = options.Scheme + } + if cacheOpts.Mapper == nil { + cacheOpts.Mapper = mapper + } + if cacheOpts.HTTPClient == nil { + cacheOpts.HTTPClient = options.HTTPClient + } + } + cache, err := options.NewCache(config, cacheOpts) if err != nil { return nil, err } - clientOptions := client.Options{Scheme: options.Scheme, Mapper: mapper} - - apiReader, err := client.New(config, clientOptions) + // Create the client, and default its options. + clientOpts := options.Client + { + if clientOpts.Scheme == nil { + clientOpts.Scheme = options.Scheme + } + if clientOpts.Mapper == nil { + clientOpts.Mapper = mapper + } + if clientOpts.HTTPClient == nil { + clientOpts.HTTPClient = options.HTTPClient + } + if clientOpts.Cache == nil { + clientOpts.Cache = &client.CacheOptions{ + Unstructured: false, + } + } + if clientOpts.Cache.Reader == nil { + clientOpts.Cache.Reader = cache + } + } + clientWriter, err := options.NewClient(config, clientOpts) if err != nil { return nil, err } - writeObj, err := options.NewClient(cache, config, clientOptions, options.ClientDisableCacheFor...) + // Create the API Reader, a client with no cache. + clientReader, err := client.New(config, client.Options{ + HTTPClient: options.HTTPClient, + Scheme: options.Scheme, + Mapper: mapper, + }) if err != nil { return nil, err } - if options.DryRunClient { - writeObj = client.NewDryRunClient(writeObj) - } - // Create the recorder provider to inject event recorders for the components. // TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific // to the particular controller that it's being injected into, rather than a generic one like is here. - recorderProvider, err := options.newRecorderProvider(config, options.Scheme, options.Logger.WithName("events"), options.makeBroadcaster) + recorderProvider, err := options.newRecorderProvider(config, options.HTTPClient, options.Scheme, options.Logger.WithName("events"), options.makeBroadcaster) if err != nil { return nil, err } return &cluster{ - config: config, + config: originalConfig, + httpClient: options.HTTPClient, scheme: options.Scheme, cache: cache, fieldIndexes: cache, - client: writeObj, - apiReader: apiReader, + client: clientWriter, + apiReader: clientReader, recorderProvider: recorderProvider, mapper: mapper, logger: options.Logger, @@ -205,21 +248,27 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { } // setOptionsDefaults set default values for Options fields. -func setOptionsDefaults(options Options) Options { +func setOptionsDefaults(options Options, config *rest.Config) (Options, error) { + if options.HTTPClient == nil { + var err error + options.HTTPClient, err = rest.HTTPClientFor(config) + if err != nil { + return options, err + } + } + // Use the Kubernetes client-go scheme if none is specified if options.Scheme == nil { options.Scheme = scheme.Scheme } if options.MapperProvider == nil { - options.MapperProvider = func(c *rest.Config) (meta.RESTMapper, error) { - return apiutil.NewDynamicRESTMapper(c) - } + options.MapperProvider = apiutil.NewDynamicRESTMapper } // Allow users to define how to create a new client if options.NewClient == nil { - options.NewClient = DefaultNewClient + options.NewClient = client.New } // Allow newCache to be mocked @@ -245,26 +294,9 @@ func setOptionsDefaults(options Options) Options { } } - if options.Logger == nil { + if options.Logger.GetSink() == nil { options.Logger = logf.RuntimeLog.WithName("cluster") } - return options -} - -// NewClientFunc allows a user to define how to create a client. -type NewClientFunc func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) - -// DefaultNewClient creates the default caching client. -func DefaultNewClient(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { - c, err := client.New(config, options) - if err != nil { - return nil, err - } - - return client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cache, - Client: c, - UncachedObjects: uncachedObjects, - }) + return options, nil } diff --git a/pkg/cluster/cluster_suite_test.go b/pkg/cluster/cluster_suite_test.go index 4970497193..dc1f9ac778 100644 --- a/pkg/cluster/cluster_suite_test.go +++ b/pkg/cluster/cluster_suite_test.go @@ -20,20 +20,18 @@ import ( "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Cluster Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Cluster Suite") } var testenv *envtest.Environment @@ -63,7 +61,7 @@ var _ = BeforeSuite(func() { clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index f9f7a0bdf3..c08a742403 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -20,20 +20,18 @@ import ( "context" "errors" "fmt" + "net/http" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/goleak" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("cluster.Cluster", func() { @@ -48,7 +46,7 @@ var _ = Describe("cluster.Cluster", func() { It("should return an error if it can't create a RestMapper", func() { expected := fmt.Errorf("expected error: RestMapper") c, err := New(cfg, func(o *Options) { - o.MapperProvider = func(c *rest.Config) (meta.RESTMapper, error) { return nil, expected } + o.MapperProvider = func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { return nil, expected } }) Expect(c).To(BeNil()) Expect(err).To(Equal(expected)) @@ -57,7 +55,7 @@ var _ = Describe("cluster.Cluster", func() { It("should return an error it can't create a client.Client", func() { c, err := New(cfg, func(o *Options) { - o.NewClient = func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + o.NewClient = func(config *rest.Config, options client.Options) (client.Client, error) { return nil, errors.New("expected error") } }) @@ -79,7 +77,7 @@ var _ = Describe("cluster.Cluster", func() { It("should create a client defined in by the new client function", func() { c, err := New(cfg, func(o *Options) { - o.NewClient = func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + o.NewClient = func(config *rest.Config, options client.Options) (client.Client, error) { return nil, nil } }) @@ -90,7 +88,7 @@ var _ = Describe("cluster.Cluster", func() { It("should return an error it can't create a recorder.Provider", func() { c, err := New(cfg, func(o *Options) { - o.newRecorderProvider = func(_ *rest.Config, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { + o.newRecorderProvider = func(_ *rest.Config, _ *http.Client, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { return nil, fmt.Errorf("expected error") } }) @@ -102,94 +100,22 @@ var _ = Describe("cluster.Cluster", func() { }) Describe("Start", func() { - It("should stop when context is cancelled", func() { + It("should stop when context is cancelled", func(specCtx SpecContext) { c, err := New(cfg) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(c.Start(ctx)).NotTo(HaveOccurred()) }) }) - Describe("SetFields", func() { - It("should inject field values", func() { - c, err := New(cfg, func(o *Options) { - o.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { - return &informertest.FakeInformers{}, nil - } - }) - Expect(err).NotTo(HaveOccurred()) - - By("Injecting the dependencies") - err = c.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - defer GinkgoRecover() - Expect(scheme).To(Equal(c.GetScheme())) - return nil - }, - config: func(config *rest.Config) error { - defer GinkgoRecover() - Expect(config).To(Equal(c.GetConfig())) - return nil - }, - client: func(client client.Client) error { - defer GinkgoRecover() - Expect(client).To(Equal(c.GetClient())) - return nil - }, - cache: func(cache cache.Cache) error { - defer GinkgoRecover() - Expect(cache).To(Equal(c.GetCache())) - return nil - }, - log: func(logger logr.Logger) error { - defer GinkgoRecover() - Expect(logger).To(Equal(logf.RuntimeLog.WithName("cluster"))) - return nil - }, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Returning an error if dependency injection fails") - - expected := fmt.Errorf("expected error") - err = c.SetFields(&injectable{ - client: func(client client.Client) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = c.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = c.SetFields(&injectable{ - config: func(config *rest.Config) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = c.SetFields(&injectable{ - cache: func(c cache.Cache) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - }) - }) - - It("should not leak goroutines when stopped", func() { + It("should not leak goroutines when stopped", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() c, err := New(cfg) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(c.Start(ctx)).NotTo(HaveOccurred()) @@ -242,56 +168,3 @@ var _ = Describe("cluster.Cluster", func() { Expect(c.GetAPIReader()).NotTo(BeNil()) }) }) - -var _ inject.Cache = &injectable{} -var _ inject.Client = &injectable{} -var _ inject.Scheme = &injectable{} -var _ inject.Config = &injectable{} -var _ inject.Logger = &injectable{} - -type injectable struct { - scheme func(scheme *runtime.Scheme) error - client func(client.Client) error - config func(config *rest.Config) error - cache func(cache.Cache) error - log func(logger logr.Logger) error -} - -func (i *injectable) InjectCache(c cache.Cache) error { - if i.cache == nil { - return nil - } - return i.cache(c) -} - -func (i *injectable) InjectConfig(config *rest.Config) error { - if i.config == nil { - return nil - } - return i.config(config) -} - -func (i *injectable) InjectClient(c client.Client) error { - if i.client == nil { - return nil - } - return i.client(c) -} - -func (i *injectable) InjectScheme(scheme *runtime.Scheme) error { - if i.scheme == nil { - return nil - } - return i.scheme(scheme) -} - -func (i *injectable) InjectLogger(log logr.Logger) error { - if i.log == nil { - return nil - } - return i.log(log) -} - -func (i *injectable) Start(<-chan struct{}) error { - return nil -} diff --git a/pkg/cluster/internal.go b/pkg/cluster/internal.go index 125e1d144e..2742764231 100644 --- a/pkg/cluster/internal.go +++ b/pkg/cluster/internal.go @@ -18,6 +18,7 @@ package cluster import ( "context" + "net/http" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/meta" @@ -28,22 +29,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) type cluster struct { // config is the rest.config used to talk to the apiserver. Required. config *rest.Config - // scheme is the scheme injected into Controllers, EventHandlers, Sources and Predicates. Defaults - // to scheme.scheme. - scheme *runtime.Scheme - - cache cache.Cache - - // TODO(directxman12): Provide an escape hatch to get individual indexers - // client is the client injected into Controllers (and EventHandlers, Sources and Predicates). - client client.Client + httpClient *http.Client + scheme *runtime.Scheme + cache cache.Cache + client client.Client // apiReader is the reader that will make requests to the api server and not the cache. apiReader client.Reader @@ -64,32 +59,14 @@ type cluster struct { logger logr.Logger } -func (c *cluster) SetFields(i interface{}) error { - if _, err := inject.ConfigInto(c.config, i); err != nil { - return err - } - if _, err := inject.ClientInto(c.client, i); err != nil { - return err - } - if _, err := inject.APIReaderInto(c.apiReader, i); err != nil { - return err - } - if _, err := inject.SchemeInto(c.scheme, i); err != nil { - return err - } - if _, err := inject.CacheInto(c.cache, i); err != nil { - return err - } - if _, err := inject.MapperInto(c.mapper, i); err != nil { - return err - } - return nil -} - func (c *cluster) GetConfig() *rest.Config { return c.config } +func (c *cluster) GetHTTPClient() *http.Client { + return c.httpClient +} + func (c *cluster) GetClient() client.Client { return c.client } diff --git a/pkg/config/config.go b/pkg/config/config.go deleted file mode 100644 index f23b02df00..0000000000 --- a/pkg/config/config.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - ioutil "io/ioutil" - "sync" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" -) - -// ControllerManagerConfiguration defines the functions necessary to parse a config file -// and to configure the Options struct for the ctrl.Manager. -type ControllerManagerConfiguration interface { - runtime.Object - - // Complete returns the versioned configuration - Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) -} - -// DeferredFileLoader is used to configure the decoder for loading controller -// runtime component config types. -type DeferredFileLoader struct { - ControllerManagerConfiguration - path string - scheme *runtime.Scheme - once sync.Once - err error -} - -// File will set up the deferred file loader for the configuration -// this will also configure the defaults for the loader if nothing is -// -// Defaults: -// Path: "./config.yaml" -// Kind: GenericControllerManagerConfiguration -func File() *DeferredFileLoader { - scheme := runtime.NewScheme() - utilruntime.Must(v1alpha1.AddToScheme(scheme)) - return &DeferredFileLoader{ - path: "./config.yaml", - ControllerManagerConfiguration: &v1alpha1.ControllerManagerConfiguration{}, - scheme: scheme, - } -} - -// Complete will use sync.Once to set the scheme. -func (d *DeferredFileLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { - d.once.Do(d.loadFile) - if d.err != nil { - return v1alpha1.ControllerManagerConfigurationSpec{}, d.err - } - return d.ControllerManagerConfiguration.Complete() -} - -// AtPath will set the path to load the file for the decoder. -func (d *DeferredFileLoader) AtPath(path string) *DeferredFileLoader { - d.path = path - return d -} - -// OfKind will set the type to be used for decoding the file into. -func (d *DeferredFileLoader) OfKind(obj ControllerManagerConfiguration) *DeferredFileLoader { - d.ControllerManagerConfiguration = obj - return d -} - -// InjectScheme will configure the scheme to be used for decoding the file. -func (d *DeferredFileLoader) InjectScheme(scheme *runtime.Scheme) error { - d.scheme = scheme - return nil -} - -// loadFile is used from the mutex.Once to load the file. -func (d *DeferredFileLoader) loadFile() { - if d.scheme == nil { - d.err = fmt.Errorf("scheme not supplied to controller configuration loader") - return - } - - content, err := ioutil.ReadFile(d.path) - if err != nil { - d.err = fmt.Errorf("could not read file at %s", d.path) - return - } - - codecs := serializer.NewCodecFactory(d.scheme) - - // Regardless of if the bytes are of any external version, - // it will be read successfully and converted into the internal version - if err = runtime.DecodeInto(codecs.UniversalDecoder(), content, d.ControllerManagerConfiguration); err != nil { - d.err = fmt.Errorf("could not decode file into runtime.Object") - } -} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go deleted file mode 100644 index f2b5461b55..0000000000 --- a/pkg/config/config_test.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" -) - -var _ = Describe("config", func() { - Describe("File", func() { - - It("should error loading from non existent file", func() { - loader := config.File() - _, err := loader.Complete() - Expect(err).ToNot(BeNil()) - }) - - It("should load a config from file", func() { - conf := v1alpha1.ControllerManagerConfiguration{} - loader := config.File().AtPath("./testdata/config.yaml").OfKind(&conf) - Expect(conf.CacheNamespace).To(Equal("")) - - _, err := loader.Complete() - Expect(err).To(BeNil()) - - Expect(*conf.LeaderElection.LeaderElect).To(Equal(true)) - Expect(conf.CacheNamespace).To(Equal("default")) - Expect(conf.Metrics.BindAddress).To(Equal(":8081")) - }) - }) -}) diff --git a/pkg/config/controller.go b/pkg/config/controller.go new file mode 100644 index 0000000000..5eea2965f6 --- /dev/null +++ b/pkg/config/controller.go @@ -0,0 +1,92 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "time" + + "github.com/go-logr/logr" +) + +// Controller contains configuration options for controllers. It only includes options +// that makes sense for a set of controllers and is used for defaulting the options +// of multiple controllers. +type Controller struct { + // SkipNameValidation allows skipping the name validation that ensures that every controller name is unique. + // Unique controller names are important to get unique metrics and logs for a controller. + // Can be overwritten for a controller via the SkipNameValidation setting on the controller. + // Defaults to false if SkipNameValidation setting on controller and Manager are unset. + SkipNameValidation *bool + + // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation + // allowed for that controller. + // + // When a controller is registered within this manager using the builder utilities, + // users have to specify the type the controller reconciles in the For(...) call. + // If the object's kind passed matches one of the keys in this map, the concurrency + // for that controller is set to the number specified. + // + // The key is expected to be consistent in form with GroupKind.String(), + // e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`. + GroupKindConcurrency map[string]int + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. + MaxConcurrentReconciles int + + // CacheSyncTimeout refers to the time limit set to wait for syncing caches. + // Defaults to 2 minutes if not set. + CacheSyncTimeout time.Duration + + // RecoverPanic indicates whether the panic caused by reconcile should be recovered. + // Can be overwritten for a controller via the RecoverPanic setting on the controller. + // Defaults to true if RecoverPanic setting on controller and Manager are unset. + RecoverPanic *bool + + // NeedLeaderElection indicates whether the controller needs to use leader election. + // Defaults to true, which means the controller will use leader election. + NeedLeaderElection *bool + + // EnableWarmup specifies whether the controller should start its sources when the manager is not + // the leader. This is useful for cases where sources take a long time to start, as it allows + // for the controller to warm up its caches even before it is elected as the leader. This + // improves leadership failover time, as the caches will be prepopulated before the controller + // transitions to be leader. + // + // Setting EnableWarmup to true and NeedLeaderElection to true means the controller will start its + // sources without waiting to become leader. + // Setting EnableWarmup to true and NeedLeaderElection to false is a no-op as controllers without + // leader election do not wait on leader election to start their sources. + // Defaults to false. + // + // Note: This feature is currently in beta and subject to change. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/3220. + EnableWarmup *bool + + // UsePriorityQueue configures the controllers queue to use the controller-runtime provided + // priority queue. + // + // Note: This flag is enabled by default. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/2374. + UsePriorityQueue *bool + + // Logger is the logger controllers should use. + Logger logr.Logger + + // ReconciliationTimeout is used as the timeout passed to the context of each Reconcile call. + // By default, there is no timeout. + ReconciliationTimeout time.Duration +} diff --git a/pkg/config/doc.go b/pkg/config/doc.go deleted file mode 100644 index ebd8243f32..0000000000 --- a/pkg/config/doc.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package config contains functionality for interacting with ComponentConfig -// files -// -// DeferredFileLoader -// -// This uses a deferred file decoding allowing you to chain your configuration -// setup. You can pass this into manager.Options#File and it will load your -// config. -package config diff --git a/pkg/config/example_test.go b/pkg/config/example_test.go deleted file mode 100644 index fb1cd58b5f..0000000000 --- a/pkg/config/example_test.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config_test - -import ( - "fmt" - "os" - - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/config" - - "sigs.k8s.io/controller-runtime/examples/configfile/custom/v1alpha1" -) - -var scheme = runtime.NewScheme() - -func init() { - _ = v1alpha1.AddToScheme(scheme) -} - -// This example will load a file using Complete with only -// defaults set. -func ExampleFile() { - // This will load a config file from ./config.yaml - loader := config.File() - if _, err := loader.Complete(); err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} - -// This example will load the file from a custom path. -func ExampleDeferredFileLoader_atPath() { - loader := config.File().AtPath("/var/run/controller-runtime/config.yaml") - if _, err := loader.Complete(); err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} - -// This example sets up loader with a custom scheme. -func ExampleDeferredFileLoader_injectScheme() { - loader := config.File() - err := loader.InjectScheme(scheme) - if err != nil { - fmt.Println("failed to inject scheme") - os.Exit(1) - } - - _, err = loader.Complete() - if err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} - -// This example sets up the loader with a custom scheme and custom type. -func ExampleDeferredFileLoader_ofKind() { - loader := config.File().OfKind(&v1alpha1.CustomControllerManagerConfiguration{}) - err := loader.InjectScheme(scheme) - if err != nil { - fmt.Println("failed to inject scheme") - os.Exit(1) - } - _, err = loader.Complete() - if err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} diff --git a/pkg/config/testdata/config.yaml b/pkg/config/testdata/config.yaml deleted file mode 100644 index d88da3a65b..0000000000 --- a/pkg/config/testdata/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfiguration -cacheNamespace: default -metrics: - bindAddress: :8081 -leaderElection: - leaderElect: true diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go deleted file mode 100644 index e67b62e514..0000000000 --- a/pkg/config/v1alpha1/types.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - configv1alpha1 "k8s.io/component-base/config/v1alpha1" -) - -// ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration. -type ControllerManagerConfigurationSpec struct { - // SyncPeriod determines the minimum frequency at which watched resources are - // reconciled. A lower period will correct entropy more quickly, but reduce - // responsiveness to change if there are many watched resources. Change this - // value only if you know what you are doing. Defaults to 10 hours if unset. - // there will a 10 percent jitter between the SyncPeriod of all controllers - // so that all controllers will not send list requests simultaneously. - // +optional - SyncPeriod *metav1.Duration `json:"syncPeriod,omitempty"` - - // LeaderElection is the LeaderElection config to be used when configuring - // the manager.Manager leader election - // +optional - LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` - - // CacheNamespace if specified restricts the manager's cache to watch objects in - // the desired namespace Defaults to all namespaces - // - // Note: If a namespace is specified, controllers can still Watch for a - // cluster-scoped resource (e.g Node). For namespaced resources the cache - // will only hold objects from the desired namespace. - // +optional - CacheNamespace string `json:"cacheNamespace,omitempty"` - - // GracefulShutdownTimeout is the duration given to runnable to stop before the manager actually returns on stop. - // To disable graceful shutdown, set to time.Duration(0) - // To use graceful shutdown without timeout, set to a negative duration, e.G. time.Duration(-1) - // The graceful shutdown is skipped for safety reasons in case the leader election lease is lost. - GracefulShutdownTimeout *metav1.Duration `json:"gracefulShutDown,omitempty"` - - // Controller contains global configuration options for controllers - // registered within this manager. - // +optional - Controller *ControllerConfigurationSpec `json:"controller,omitempty"` - - // Metrics contains thw controller metrics configuration - // +optional - Metrics ControllerMetrics `json:"metrics,omitempty"` - - // Health contains the controller health configuration - // +optional - Health ControllerHealth `json:"health,omitempty"` - - // Webhook contains the controllers webhook configuration - // +optional - Webhook ControllerWebhook `json:"webhook,omitempty"` -} - -// ControllerConfigurationSpec defines the global configuration for -// controllers registered with the manager. -type ControllerConfigurationSpec struct { - // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation - // allowed for that controller. - // - // When a controller is registered within this manager using the builder utilities, - // users have to specify the type the controller reconciles in the For(...) call. - // If the object's kind passed matches one of the keys in this map, the concurrency - // for that controller is set to the number specified. - // - // The key is expected to be consistent in form with GroupKind.String(), - // e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`. - // - // +optional - GroupKindConcurrency map[string]int `json:"groupKindConcurrency,omitempty"` - - // CacheSyncTimeout refers to the time limit set to wait for syncing caches. - // Defaults to 2 minutes if not set. - // +optional - CacheSyncTimeout *time.Duration `json:"cacheSyncTimeout,omitempty"` -} - -// ControllerMetrics defines the metrics configs. -type ControllerMetrics struct { - // BindAddress is the TCP address that the controller should bind to - // for serving prometheus metrics. - // It can be set to "0" to disable the metrics serving. - // +optional - BindAddress string `json:"bindAddress,omitempty"` -} - -// ControllerHealth defines the health configs. -type ControllerHealth struct { - // HealthProbeBindAddress is the TCP address that the controller should bind to - // for serving health probes - // +optional - HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"` - - // ReadinessEndpointName, defaults to "readyz" - // +optional - ReadinessEndpointName string `json:"readinessEndpointName,omitempty"` - - // LivenessEndpointName, defaults to "healthz" - // +optional - LivenessEndpointName string `json:"livenessEndpointName,omitempty"` -} - -// ControllerWebhook defines the webhook server for the controller. -type ControllerWebhook struct { - // Port is the port that the webhook server serves at. - // It is used to set webhook.Server.Port. - // +optional - Port *int `json:"port,omitempty"` - - // Host is the hostname that the webhook server binds to. - // It is used to set webhook.Server.Host. - // +optional - Host string `json:"host,omitempty"` - - // CertDir is the directory that contains the server key and certificate. - // if not set, webhook server would look up the server key and certificate in - // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate - // must be named tls.key and tls.crt, respectively. - // +optional - CertDir string `json:"certDir,omitempty"` -} - -// +kubebuilder:object:root=true - -// ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API. -type ControllerManagerConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // ControllerManagerConfiguration returns the contfigurations for controllers - ControllerManagerConfigurationSpec `json:",inline"` -} - -// Complete returns the configuration for controller-runtime. -func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) { - return *c, nil -} diff --git a/pkg/config/v1alpha1/zz_generated.deepcopy.go b/pkg/config/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index 752fa9754c..0000000000 --- a/pkg/config/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,152 +0,0 @@ -// +build !ignore_autogenerated - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - configv1alpha1 "k8s.io/component-base/config/v1alpha1" - timex "time" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerConfigurationSpec) DeepCopyInto(out *ControllerConfigurationSpec) { - *out = *in - if in.GroupKindConcurrency != nil { - in, out := &in.GroupKindConcurrency, &out.GroupKindConcurrency - *out = make(map[string]int, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.CacheSyncTimeout != nil { - in, out := &in.CacheSyncTimeout, &out.CacheSyncTimeout - *out = new(timex.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerConfigurationSpec. -func (in *ControllerConfigurationSpec) DeepCopy() *ControllerConfigurationSpec { - if in == nil { - return nil - } - out := new(ControllerConfigurationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth. -func (in *ControllerHealth) DeepCopy() *ControllerHealth { - if in == nil { - return nil - } - out := new(ControllerHealth) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerManagerConfiguration) DeepCopyInto(out *ControllerManagerConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfiguration. -func (in *ControllerManagerConfiguration) DeepCopy() *ControllerManagerConfiguration { - if in == nil { - return nil - } - out := new(ControllerManagerConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ControllerManagerConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerManagerConfigurationSpec) DeepCopyInto(out *ControllerManagerConfigurationSpec) { - *out = *in - if in.SyncPeriod != nil { - in, out := &in.SyncPeriod, &out.SyncPeriod - *out = new(v1.Duration) - **out = **in - } - if in.LeaderElection != nil { - in, out := &in.LeaderElection, &out.LeaderElection - *out = new(configv1alpha1.LeaderElectionConfiguration) - (*in).DeepCopyInto(*out) - } - if in.GracefulShutdownTimeout != nil { - in, out := &in.GracefulShutdownTimeout, &out.GracefulShutdownTimeout - *out = new(v1.Duration) - **out = **in - } - if in.Controller != nil { - in, out := &in.Controller, &out.Controller - *out = new(ControllerConfigurationSpec) - (*in).DeepCopyInto(*out) - } - out.Metrics = in.Metrics - out.Health = in.Health - in.Webhook.DeepCopyInto(&out.Webhook) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfigurationSpec. -func (in *ControllerManagerConfigurationSpec) DeepCopy() *ControllerManagerConfigurationSpec { - if in == nil { - return nil - } - out := new(ControllerManagerConfigurationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics. -func (in *ControllerMetrics) DeepCopy() *ControllerMetrics { - if in == nil { - return nil - } - out := new(ControllerMetrics) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) { - *out = *in - if in.Port != nil { - in, out := &in.Port, &out.Port - *out = new(int) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook. -func (in *ControllerWebhook) DeepCopy() *ControllerWebhook { - if in == nil { - return nil - } - out := new(ControllerWebhook) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 88ba786717..853788d52f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -23,55 +23,151 @@ import ( "github.com/go-logr/logr" "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/handler" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/internal/controller" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/ratelimiter" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) // Options are the arguments for creating a new Controller. -type Options struct { +type Options = TypedOptions[reconcile.Request] + +// TypedOptions are the arguments for creating a new Controller. +type TypedOptions[request comparable] struct { + // SkipNameValidation allows skipping the name validation that ensures that every controller name is unique. + // Unique controller names are important to get unique metrics and logs for a controller. + // Defaults to the Controller.SkipNameValidation setting from the Manager if unset. + // Defaults to false if Controller.SkipNameValidation setting from the Manager is also unset. + SkipNameValidation *bool + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. MaxConcurrentReconciles int + // CacheSyncTimeout refers to the time limit set to wait for syncing caches. + // Defaults to 2 minutes if not set. + CacheSyncTimeout time.Duration + + // RecoverPanic indicates whether the panic caused by reconcile should be recovered. + // Defaults to the Controller.RecoverPanic setting from the Manager if unset. + // Defaults to true if Controller.RecoverPanic setting from the Manager is also unset. + RecoverPanic *bool + + // NeedLeaderElection indicates whether the controller needs to use leader election. + // Defaults to true, which means the controller will use leader election. + NeedLeaderElection *bool + // Reconciler reconciles an object - Reconciler reconcile.Reconciler + Reconciler reconcile.TypedReconciler[request] // RateLimiter is used to limit how frequently requests may be queued. // Defaults to MaxOfRateLimiter which has both overall and per-item rate limiting. // The overall is a token bucket and the per-item is exponential. - RateLimiter ratelimiter.RateLimiter + RateLimiter workqueue.TypedRateLimiter[request] - // Log is the logger used for this controller and passed to each reconciliation - // request via the context field. - Log logr.Logger + // NewQueue constructs the queue for this controller once the controller is ready to start. + // With NewQueue a custom queue implementation can be used, e.g. a priority queue to prioritize with which + // priority/order objects are reconciled (e.g. to reconcile objects with changes first). + // This is a func because the standard Kubernetes work queues start themselves immediately, which + // leads to goroutine leaks if something calls controller.New repeatedly. + // The NewQueue func gets the controller name and the RateLimiter option (defaulted if necessary) passed in. + // NewQueue defaults to NewRateLimitingQueueWithConfig. + // + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewQueue if you know what you are doing. + NewQueue func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] - // CacheSyncTimeout refers to the time limit set to wait for syncing caches. - // Defaults to 2 minutes if not set. - CacheSyncTimeout time.Duration + // Logger will be used to build a default LogConstructor if unset. + Logger logr.Logger - // RecoverPanic indicates whether the panic caused by reconcile should be recovered. - RecoverPanic bool + // LogConstructor is used to construct a logger used for this controller and passed + // to each reconciliation via the context field. + LogConstructor func(request *request) logr.Logger + + // UsePriorityQueue configures the controllers queue to use the controller-runtime provided + // priority queue. + // + // Note: This flag is enabled by default. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/2374. + UsePriorityQueue *bool + + // EnableWarmup specifies whether the controller should start its sources when the manager is not + // the leader. This is useful for cases where sources take a long time to start, as it allows + // for the controller to warm up its caches even before it is elected as the leader. This + // improves leadership failover time, as the caches will be prepopulated before the controller + // transitions to be leader. + // + // Setting EnableWarmup to true and NeedLeaderElection to true means the controller will start its + // sources without waiting to become leader. + // Setting EnableWarmup to true and NeedLeaderElection to false is a no-op as controllers without + // leader election do not wait on leader election to start their sources. + // Defaults to false. + // + // Note: This feature is currently in beta and subject to change. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/3220. + EnableWarmup *bool + + // ReconciliationTimeout is used as the timeout passed to the context of each Reconcile call. + // By default, there is no timeout. + ReconciliationTimeout time.Duration +} + +// DefaultFromConfig defaults the config from a config.Controller +func (options *TypedOptions[request]) DefaultFromConfig(config config.Controller) { + if options.Logger.GetSink() == nil { + options.Logger = config.Logger + } + + if options.SkipNameValidation == nil { + options.SkipNameValidation = config.SkipNameValidation + } + + if options.MaxConcurrentReconciles <= 0 && config.MaxConcurrentReconciles > 0 { + options.MaxConcurrentReconciles = config.MaxConcurrentReconciles + } + + if options.CacheSyncTimeout == 0 && config.CacheSyncTimeout > 0 { + options.CacheSyncTimeout = config.CacheSyncTimeout + } + + if options.UsePriorityQueue == nil { + options.UsePriorityQueue = config.UsePriorityQueue + } + + if options.RecoverPanic == nil { + options.RecoverPanic = config.RecoverPanic + } + + if options.NeedLeaderElection == nil { + options.NeedLeaderElection = config.NeedLeaderElection + } + + if options.EnableWarmup == nil { + options.EnableWarmup = config.EnableWarmup + } + + if options.ReconciliationTimeout == 0 { + options.ReconciliationTimeout = config.ReconciliationTimeout + } } -// Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests -// from source.Sources. Work is performed through the reconcile.Reconciler for each enqueued item. +// Controller implements an API. A Controller manages a work queue fed reconcile.Requests +// from source.Sources. Work is performed through the reconcile.Reconciler for each enqueued item. // Work typically is reads and writes Kubernetes objects to make the system state match the state specified // in the object Spec. -type Controller interface { +type Controller = TypedController[reconcile.Request] + +// TypedController implements an API. +type TypedController[request comparable] interface { // Reconciler is called to reconcile an object by Namespace/Name - reconcile.Reconciler + reconcile.TypedReconciler[request] - // Watch takes events provided by a Source and uses the EventHandler to - // enqueue reconcile.Requests in response to the events. - // - // Watch may be provided one or more Predicates to filter events before - // they are given to the EventHandler. Events will be passed to the - // EventHandler if all provided Predicates evaluate to true. - Watch(src source.Source, eventhandler handler.EventHandler, predicates ...predicate.Predicate) error + // Watch watches the provided Source. + Watch(src source.TypedSource[request]) error // Start starts the controller. Start blocks until the context is closed or a // controller has an error starting. @@ -83,8 +179,18 @@ type Controller interface { // New returns a new Controller registered with the Manager. The Manager will ensure that shared Caches have // been synced before the Controller is Started. +// +// The name must be unique as it is used to identify the controller in metrics and logs. func New(name string, mgr manager.Manager, options Options) (Controller, error) { - c, err := NewUnmanaged(name, mgr, options) + return NewTyped(name, mgr, options) +} + +// NewTyped returns a new typed controller registered with the Manager, +// +// The name must be unique as it is used to identify the controller in metrics and logs. +func NewTyped[request comparable](name string, mgr manager.Manager, options TypedOptions[request]) (TypedController[request], error) { + options.DefaultFromConfig(mgr.GetControllerOptions()) + c, err := NewTypedUnmanaged(name, options) if err != nil { return nil, err } @@ -95,7 +201,16 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error) // NewUnmanaged returns a new controller without adding it to the manager. The // caller is responsible for starting the returned controller. -func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller, error) { +// +// The name must be unique as it is used to identify the controller in metrics and logs. +func NewUnmanaged(name string, options Options) (Controller, error) { + return NewTypedUnmanaged(name, options) +} + +// NewTypedUnmanaged returns a new typed controller without adding it to the manager. +// +// The name must be unique as it is used to identify the controller in metrics and logs. +func NewTypedUnmanaged[request comparable](name string, options TypedOptions[request]) (TypedController[request], error) { if options.Reconciler == nil { return nil, fmt.Errorf("must specify Reconciler") } @@ -104,8 +219,26 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller return nil, fmt.Errorf("must specify Name for Controller") } - if options.Log == nil { - options.Log = mgr.GetLogger() + if options.SkipNameValidation == nil || !*options.SkipNameValidation { + if err := checkName(name); err != nil { + return nil, err + } + } + + if options.LogConstructor == nil { + log := options.Logger.WithValues( + "controller", name, + ) + options.LogConstructor = func(in *request) logr.Logger { + log := log + if req, ok := any(in).(*reconcile.Request); ok && req != nil { + log = log.WithValues( + "object", klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + ) + } + return log + } } if options.MaxConcurrentReconciles <= 0 { @@ -117,25 +250,42 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller } if options.RateLimiter == nil { - options.RateLimiter = workqueue.DefaultControllerRateLimiter() + if ptr.Deref(options.UsePriorityQueue, true) { + options.RateLimiter = workqueue.NewTypedItemExponentialFailureRateLimiter[request](5*time.Millisecond, 1000*time.Second) + } else { + options.RateLimiter = workqueue.DefaultTypedControllerRateLimiter[request]() + } } - // Inject dependencies into Reconciler - if err := mgr.SetFields(options.Reconciler); err != nil { - return nil, err + if options.NewQueue == nil { + options.NewQueue = func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] { + if ptr.Deref(options.UsePriorityQueue, true) { + return priorityqueue.New(controllerName, func(o *priorityqueue.Opts[request]) { + o.Log = options.Logger.WithValues("controller", controllerName) + o.RateLimiter = rateLimiter + }) + } + return workqueue.NewTypedRateLimitingQueueWithConfig(rateLimiter, workqueue.TypedRateLimitingQueueConfig[request]{ + Name: controllerName, + }) + } } // Create controller with dependencies set - return &controller.Controller{ - Do: options.Reconciler, - MakeQueue: func() workqueue.RateLimitingInterface { - return workqueue.NewNamedRateLimitingQueue(options.RateLimiter, name) - }, + return controller.New[request](controller.Options[request]{ + Do: options.Reconciler, + RateLimiter: options.RateLimiter, + NewQueue: options.NewQueue, MaxConcurrentReconciles: options.MaxConcurrentReconciles, CacheSyncTimeout: options.CacheSyncTimeout, - SetFields: mgr.SetFields, Name: name, - Log: options.Log.WithName("controller").WithName(name), + LogConstructor: options.LogConstructor, RecoverPanic: options.RecoverPanic, - }, nil + LeaderElected: options.NeedLeaderElection, + EnableWarmup: options.EnableWarmup, + ReconciliationTimeout: options.ReconciliationTimeout, + }), nil } + +// ReconcileIDFromContext gets the reconcileID from the current context. +var ReconcileIDFromContext = controller.ReconcileIDFromContext diff --git a/pkg/controller/controller_integration_test.go b/pkg/controller/controller_integration_test.go index 9f347b0032..e09813eee2 100644 --- a/pkg/controller/controller_integration_test.go +++ b/pkg/controller/controller_integration_test.go @@ -18,27 +18,31 @@ package controller_test import ( "context" + "fmt" + "strconv" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/manager" ) var _ = Describe("controller", func() { var reconciled chan reconcile.Request - ctx := context.Background() BeforeEach(func() { reconciled = make(chan reconcile.Request) @@ -48,28 +52,48 @@ var _ = Describe("controller", func() { Describe("controller", func() { // TODO(directxman12): write a whole suite of controller-client interaction tests - It("should reconcile", func() { + // The watches in this test are setup with a namespace predicate to avoid each table entry + // from interfering with the others. We cannot add a delete call for the pods created in the + // test, as it causes flakes with the api-server termination timing out. + // See https://github.com/kubernetes-sigs/controller-runtime/issues/1571 for a description + // of the issue, and a discussion here: https://github.com/kubernetes-sigs/controller-runtime/pull/3192#discussion_r2186967799 + DescribeTable("should reconcile", func(ctx SpecContext, enableWarmup bool) { By("Creating the Manager") cm, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) By("Creating the Controller") - instance, err := controller.New("foo-controller", cm, controller.Options{ - Reconciler: reconcile.Func( - func(_ context.Context, request reconcile.Request) (reconcile.Result, error) { - reconciled <- request - return reconcile.Result{}, nil - }), - }) + instance, err := controller.New( + fmt.Sprintf("foo-controller-%t", enableWarmup), + cm, + controller.Options{ + Reconciler: reconcile.Func( + func(_ context.Context, request reconcile.Request) (reconcile.Result, error) { + reconciled <- request + return reconcile.Result{}, nil + }), + EnableWarmup: ptr.To(enableWarmup), + }, + ) Expect(err).NotTo(HaveOccurred()) + testNamespace := strconv.FormatBool(enableWarmup) + By("Watching Resources") - err = instance.Watch(&source.Kind{Type: &appsv1.ReplicaSet{}}, &handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.Deployment{}, - }) + err = instance.Watch( + source.Kind(cm.GetCache(), &appsv1.ReplicaSet{}, + handler.TypedEnqueueRequestForOwner[*appsv1.ReplicaSet](cm.GetScheme(), cm.GetRESTMapper(), &appsv1.Deployment{}), + makeNamespacePredicate[*appsv1.ReplicaSet](testNamespace), + ), + ) Expect(err).NotTo(HaveOccurred()) - err = instance.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForObject{}) + err = instance.Watch( + source.Kind(cm.GetCache(), &appsv1.Deployment{}, + &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{}, + makeNamespacePredicate[*appsv1.Deployment](testNamespace), + ), + ) Expect(err).NotTo(HaveOccurred()) err = cm.GetClient().Get(ctx, types.NamespacedName{Name: "foo"}, &corev1.Namespace{}) @@ -78,8 +102,6 @@ var _ = Describe("controller", func() { Expect(err).To(Equal(&cache.ErrCacheNotStarted{})) By("Starting the Manager") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(cm.Start(ctx)).NotTo(HaveOccurred()) @@ -108,19 +130,25 @@ var _ = Describe("controller", func() { }, } expectedReconcileRequest := reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: "default", + Namespace: testNamespace, Name: "deployment-name", }} + By("Creating the test namespace") + _, err = clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: testNamespace}, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Invoking Reconciling for Create") - deployment, err = clientset.AppsV1().Deployments("default").Create(ctx, deployment, metav1.CreateOptions{}) + deployment, err = clientset.AppsV1().Deployments(testNamespace).Create(ctx, deployment, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for Update") newDeployment := deployment.DeepCopy() newDeployment.Labels = map[string]string{"foo": "bar"} - _, err = clientset.AppsV1().Deployments("default").Update(ctx, newDeployment, metav1.UpdateOptions{}) + _, err = clientset.AppsV1().Deployments(testNamespace).Update(ctx, newDeployment, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) @@ -143,36 +171,81 @@ var _ = Describe("controller", func() { Template: deployment.Spec.Template, }, } - replicaset, err = clientset.AppsV1().ReplicaSets("default").Create(ctx, replicaset, metav1.CreateOptions{}) + replicaset, err = clientset.AppsV1().ReplicaSets(testNamespace).Create(ctx, replicaset, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for an OwnedObject when it is updated") newReplicaset := replicaset.DeepCopy() newReplicaset.Labels = map[string]string{"foo": "bar"} - _, err = clientset.AppsV1().ReplicaSets("default").Update(ctx, newReplicaset, metav1.UpdateOptions{}) + _, err = clientset.AppsV1().ReplicaSets(testNamespace).Update(ctx, newReplicaset, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for an OwnedObject when it is deleted") - err = clientset.AppsV1().ReplicaSets("default").Delete(ctx, replicaset.Name, metav1.DeleteOptions{}) + err = clientset.AppsV1().ReplicaSets(testNamespace).Delete(ctx, replicaset.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for Delete") - err = clientset.AppsV1().Deployments("default"). + err = clientset.AppsV1().Deployments(testNamespace). Delete(ctx, "deployment-name", metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Listing a type with a slice of pointers as items field") err = cm.GetClient(). - List(context.Background(), &controllertest.UnconventionalListTypeList{}) + List(ctx, &controllertest.UnconventionalListTypeList{}) Expect(err).NotTo(HaveOccurred()) - }, 5) + + By("Invoking Reconciling for a pod when it is created when adding watcher dynamically") + // Add new watcher dynamically + err = instance.Watch( + source.Kind(cm.GetCache(), &corev1.Pod{}, + &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}, + makeNamespacePredicate[*corev1.Pod](testNamespace), + ), + ) + Expect(err).NotTo(HaveOccurred()) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-name"}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:latest", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 80, + }, + }, + }, + }, + }, + } + expectedReconcileRequest = reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: testNamespace, + Name: "pod-name", + }} + _, err = clientset.CoreV1().Pods(testNamespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(<-reconciled).To(Equal(expectedReconcileRequest)) + }, + Entry("with controller warmup enabled", true), + Entry("with controller warmup not enabled", false), + ) }) }) +// makeNamespacePredicate returns a predicate that filters out all objects not in the passed in +// namespace. +func makeNamespacePredicate[object client.Object](namespace string) predicate.TypedPredicate[object] { + return predicate.NewTypedPredicateFuncs[object](func(obj object) bool { + return obj.GetNamespace() == namespace + }) +} + func truePtr() *bool { t := true return &t diff --git a/pkg/controller/controller_suite_test.go b/pkg/controller/controller_suite_test.go index 71b2232239..57e5471d03 100644 --- a/pkg/controller/controller_suite_test.go +++ b/pkg/controller/controller_suite_test.go @@ -20,26 +20,24 @@ import ( "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics" crscheme "sigs.k8s.io/controller-runtime/pkg/scheme" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Controller Integration Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Controller Integration Suite") } var testenv *envtest.Environment @@ -59,7 +57,7 @@ var _ = BeforeSuite(func() { &controllertest.UnconventionalListType{}, &controllertest.UnconventionalListTypeList{}, ).AddToScheme(scheme.Scheme) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) testenv = &envtest.Environment{ CRDDirectoryPaths: []string{"testdata/crds"}, @@ -81,12 +79,12 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) // Prevent the metrics listener being created - metrics.DefaultBindAddress = "0" -}, 60) + metricsserver.DefaultBindAddress = "0" +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) // Put the DefaultBindAddress back - metrics.DefaultBindAddress = ":8080" + metricsserver.DefaultBindAddress = ":8080" }) diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index d3e8419a16..06138a476b 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -18,21 +18,23 @@ package controller_test import ( "context" - "fmt" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/goleak" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + internalcontroller "sigs.k8s.io/controller-runtime/pkg/internal/controller" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" ) @@ -59,14 +61,66 @@ var _ = Describe("controller.Controller", func() { Expect(err.Error()).To(ContainSubstring("must specify Reconciler")) }) - It("NewController should return an error if injecting Reconciler fails", func() { + It("should return an error if two controllers are registered with the same name", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("foo", m, controller.Options{Reconciler: &failRec{}}) - Expect(c).To(BeNil()) + c1, err := controller.New("c3", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c3", m, controller.Options{Reconciler: rec}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("controller with name c3 already exists")) + Expect(c2).To(BeNil()) + }) + + It("should return an error if two controllers are registered with the same name and SkipNameValidation is set to false on the manager", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ + SkipNameValidation: ptr.To(false), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + c1, err := controller.New("c4", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c4", m, controller.Options{Reconciler: rec}) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("expected error")) + Expect(err.Error()).To(ContainSubstring("controller with name c4 already exists")) + Expect(c2).To(BeNil()) + }) + + It("should not return an error if two controllers are registered with the same name and SkipNameValidation is set on the manager", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ + SkipNameValidation: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + c1, err := controller.New("c5", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c5", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c2).ToNot(BeNil()) + }) + + It("should not return an error if two controllers are registered with the same name and SkipNameValidation is set on the controller", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c1, err := controller.New("c6", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c6", m, controller.Options{Reconciler: rec, SkipNameValidation: ptr.To(true)}) + Expect(err).NotTo(HaveOccurred()) + Expect(c2).ToNot(BeNil()) }) It("should not return an error if two controllers are registered with different names", func() { @@ -82,11 +136,12 @@ var _ = Describe("controller.Controller", func() { Expect(c2).ToNot(BeNil()) }) - It("should not leak goroutines when stopped", func() { + It("should not leak goroutines when stopped", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() + ctx, cancel := context.WithCancel(specCtx) watchChan := make(chan event.GenericEvent, 1) - watch := &source.Channel{Source: watchChan} + watch := source.Channel(watchChan, &handler.EnqueueRequestForObject{}) watchChan <- event.GenericEvent{Object: &corev1.Pod{}} reconcileStarted := make(chan struct{}) @@ -107,11 +162,10 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{Reconciler: rec}) - Expect(c.Watch(watch, &handler.EnqueueRequestForObject{})).To(Succeed()) + c, err := controller.New("new-controller-0", m, controller.Options{Reconciler: rec}) + Expect(c.Watch(watch)).To(Succeed()) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).To(Succeed()) @@ -134,7 +188,7 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - _, err = controller.New("new-controller", m, controller.Options{Reconciler: rec}) + _, err = controller.New("new-controller-1", m, controller.Options{Reconciler: rec}) Expect(err).NotTo(HaveOccurred()) // force-close keep-alive connections. These'll time anyway (after @@ -142,18 +196,385 @@ var _ = Describe("controller.Controller", func() { clientTransport.CloseIdleConnections() Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) }) - }) -}) -var _ reconcile.Reconciler = &failRec{} -var _ inject.Client = &failRec{} + It("should default RateLimiter and NewQueue if not specified", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-2", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.RateLimiter).NotTo(BeNil()) + Expect(ctrl.NewQueue).NotTo(BeNil()) + }) + + It("should not override RateLimiter and NewQueue if specified", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + customRateLimiter := workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](5*time.Millisecond, 1000*time.Second) + customNewQueueCalled := false + customNewQueue := func(controllerName string, rateLimiter workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + customNewQueueCalled = true + return nil + } + + c, err := controller.New("new-controller-3", m, controller.Options{ + Reconciler: reconcile.Func(nil), + RateLimiter: customRateLimiter, + NewQueue: customNewQueue, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.RateLimiter).To(BeIdenticalTo(customRateLimiter)) + ctrl.NewQueue("controller1", nil) + Expect(customNewQueueCalled).To(BeTrue(), "Expected customNewQueue to be called") + }) + + It("should default RecoverPanic from the manager", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: ptr.To(true)}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-4", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.RecoverPanic).NotTo(BeNil()) + Expect(*ctrl.RecoverPanic).To(BeTrue()) + }) + + It("should not override RecoverPanic on the controller", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: ptr.To(true)}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller", m, controller.Options{ + RecoverPanic: ptr.To(false), + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.RecoverPanic).NotTo(BeNil()) + Expect(*ctrl.RecoverPanic).To(BeFalse()) + }) + + It("should default NeedLeaderElection from the manager", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{NeedLeaderElection: ptr.To(true)}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-5", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.NeedLeaderElection()).To(BeTrue()) + }) + + It("should not override NeedLeaderElection on the controller", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{NeedLeaderElection: ptr.To(true)}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-6", m, controller.Options{ + NeedLeaderElection: ptr.To(false), + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.NeedLeaderElection()).To(BeFalse()) + }) + + It("Should default MaxConcurrentReconciles from the manager if set", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{MaxConcurrentReconciles: 5}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-7", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.MaxConcurrentReconciles).To(BeEquivalentTo(5)) + }) + + It("Should default MaxConcurrentReconciles to 1 if unset", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-8", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.MaxConcurrentReconciles).To(BeEquivalentTo(1)) + }) + + It("Should leave MaxConcurrentReconciles if set", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-9", m, controller.Options{ + Reconciler: reconcile.Func(nil), + MaxConcurrentReconciles: 5, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.MaxConcurrentReconciles).To(BeEquivalentTo(5)) + }) + + It("Should default CacheSyncTimeout from the manager if set", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{CacheSyncTimeout: 5}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-10", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.CacheSyncTimeout).To(BeEquivalentTo(5)) + }) + + It("Should default CacheSyncTimeout to 2 minutes if unset", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-11", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.CacheSyncTimeout).To(BeEquivalentTo(2 * time.Minute)) + }) + + It("Should leave CacheSyncTimeout if set", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-12", m, controller.Options{ + Reconciler: reconcile.Func(nil), + CacheSyncTimeout: 5, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.CacheSyncTimeout).To(BeEquivalentTo(5)) + }) + + It("should default NeedLeaderElection on the controller to true", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-13", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.NeedLeaderElection()).To(BeTrue()) + }) + + It("should allow for setting leaderElected to false", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-14", m, controller.Options{ + NeedLeaderElection: ptr.To(false), + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.NeedLeaderElection()).To(BeFalse()) + }) + + It("should implement manager.LeaderElectionRunnable", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-15", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + _, ok := c.(manager.LeaderElectionRunnable) + Expect(ok).To(BeTrue()) + }) + + It("should configure a priority queue per default", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{}, + }) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-16", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + q := ctrl.NewQueue("foo", nil) + _, ok = q.(priorityqueue.PriorityQueue[reconcile.Request]) + Expect(ok).To(BeTrue()) + }) + + It("should not configure a priority queue if UsePriorityQueue is set to false", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-17", m, controller.Options{ + Reconciler: rec, + UsePriorityQueue: ptr.To(false), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + q := ctrl.NewQueue("foo", nil) + _, ok = q.(priorityqueue.PriorityQueue[reconcile.Request]) + Expect(ok).To(BeFalse()) + }) + + It("should set EnableWarmup correctly", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + // Test with EnableWarmup set to true + ctrlWithWarmup, err := controller.New("warmup-enabled-ctrl", m, controller.Options{ + Reconciler: reconcile.Func(nil), + EnableWarmup: ptr.To(true), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlWithWarmup, ok := ctrlWithWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlWithWarmup.EnableWarmup).To(HaveValue(BeTrue())) + + // Test with EnableWarmup set to false + ctrlWithoutWarmup, err := controller.New("warmup-disabled-ctrl", m, controller.Options{ + Reconciler: reconcile.Func(nil), + EnableWarmup: ptr.To(false), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlWithoutWarmup, ok := ctrlWithoutWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlWithoutWarmup.EnableWarmup).To(HaveValue(BeFalse())) + + // Test with EnableWarmup not set (should default to nil) + ctrlWithDefaultWarmup, err := controller.New("warmup-default-ctrl", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlWithDefaultWarmup, ok := ctrlWithDefaultWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlWithDefaultWarmup.EnableWarmup).To(BeNil()) + }) + + It("should inherit EnableWarmup from manager config", func() { + // Test with manager default setting EnableWarmup to true + managerWithWarmup, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ + EnableWarmup: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + ctrlInheritingWarmup, err := controller.New("inherit-warmup-enabled", managerWithWarmup, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlInheritingWarmup, ok := ctrlInheritingWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlInheritingWarmup.EnableWarmup).To(HaveValue(BeTrue())) + + // Test that explicit controller setting overrides manager setting + ctrlOverridingWarmup, err := controller.New("override-warmup-disabled", managerWithWarmup, controller.Options{ + Reconciler: reconcile.Func(nil), + EnableWarmup: ptr.To(false), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlOverridingWarmup, ok := ctrlOverridingWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlOverridingWarmup.EnableWarmup).To(HaveValue(BeFalse())) + }) + + It("should default ReconciliationTimeout from manager if unset", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ReconciliationTimeout: 30 * time.Second}, + }) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("mgr-reconciliation-timeout", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.ReconciliationTimeout).To(Equal(30 * time.Second)) + }) + + It("should not override an existing ReconciliationTimeout", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ReconciliationTimeout: 30 * time.Second}, + }) + Expect(err).NotTo(HaveOccurred()) -type failRec struct{} + c, err := controller.New("ctrl-reconciliation-timeout", m, controller.Options{ + Reconciler: rec, + ReconciliationTimeout: time.Minute, + }) + Expect(err).NotTo(HaveOccurred()) -func (*failRec) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, nil -} + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) -func (*failRec) InjectClient(client.Client) error { - return fmt.Errorf("expected error") -} + Expect(ctrl.ReconciliationTimeout).To(Equal(time.Minute)) + }) + }) +}) diff --git a/pkg/controller/controllertest/testing.go b/pkg/controller/controllertest/testing.go index b9f97d5289..2b481d2116 100644 --- a/pkg/controller/controllertest/testing.go +++ b/pkg/controller/controllertest/testing.go @@ -17,11 +17,13 @@ limitations under the License. package controllertest import ( + "sync" "time" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var _ runtime.Object = &ErrorType{} @@ -35,28 +37,37 @@ func (ErrorType) GetObjectKind() schema.ObjectKind { return nil } // DeepCopyObject implements runtime.Object. func (ErrorType) DeepCopyObject() runtime.Object { return nil } -var _ workqueue.RateLimitingInterface = Queue{} +var _ workqueue.TypedRateLimitingInterface[reconcile.Request] = &Queue{} // Queue implements a RateLimiting queue as a non-ratelimited queue for testing. // This helps testing by having functions that use a RateLimiting queue synchronously add items to the queue. -type Queue struct { - workqueue.Interface +type Queue = TypedQueue[reconcile.Request] + +// TypedQueue implements a RateLimiting queue as a non-ratelimited queue for testing. +// This helps testing by having functions that use a RateLimiting queue synchronously add items to the queue. +type TypedQueue[request comparable] struct { + workqueue.TypedInterface[request] + AddedRateLimitedLock sync.Mutex + AddedRatelimited []any } // AddAfter implements RateLimitingInterface. -func (q Queue) AddAfter(item interface{}, duration time.Duration) { +func (q *TypedQueue[request]) AddAfter(item request, duration time.Duration) { q.Add(item) } // AddRateLimited implements RateLimitingInterface. TODO(community): Implement this. -func (q Queue) AddRateLimited(item interface{}) { +func (q *TypedQueue[request]) AddRateLimited(item request) { + q.AddedRateLimitedLock.Lock() + q.AddedRatelimited = append(q.AddedRatelimited, item) + q.AddedRateLimitedLock.Unlock() q.Add(item) } // Forget implements RateLimitingInterface. TODO(community): Implement this. -func (q Queue) Forget(item interface{}) {} +func (q *TypedQueue[request]) Forget(item request) {} // NumRequeues implements RateLimitingInterface. TODO(community): Implement this. -func (q Queue) NumRequeues(item interface{}) int { +func (q *TypedQueue[request]) NumRequeues(item request) int { return 0 } diff --git a/pkg/controller/controllertest/util.go b/pkg/controller/controllertest/util.go index df6ae9a223..2c9a248899 100644 --- a/pkg/controller/controllertest/util.go +++ b/pkg/controller/controllertest/util.go @@ -17,6 +17,7 @@ limitations under the License. package controllertest import ( + "context" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -56,9 +57,22 @@ func (f *FakeInformer) HasSynced() bool { return f.Synced } -// AddEventHandler implements the Informer interface. Adds an EventHandler to the fake Informers. -func (f *FakeInformer) AddEventHandler(handler cache.ResourceEventHandler) { +// AddEventHandler implements the Informer interface. Adds an EventHandler to the fake Informers. TODO(community): Implement Registration. +func (f *FakeInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) { f.handlers = append(f.handlers, handler) + return nil, nil +} + +// AddEventHandlerWithResyncPeriod implements the Informer interface. Adds an EventHandler to the fake Informers (ignores resyncPeriod). TODO(community): Implement Registration. +func (f *FakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, _ time.Duration) (cache.ResourceEventHandlerRegistration, error) { + f.handlers = append(f.handlers, handler) + return nil, nil +} + +// AddEventHandlerWithOptions implements the Informer interface. Adds an EventHandler to the fake Informers (ignores options). TODO(community): Implement Registration. +func (f *FakeInformer) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, _ cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + f.handlers = append(f.handlers, handler) + return nil, nil } // Run implements the Informer interface. Increments f.RunCount. @@ -66,10 +80,14 @@ func (f *FakeInformer) Run(<-chan struct{}) { f.RunCount++ } +func (f *FakeInformer) RunWithContext(_ context.Context) { + f.RunCount++ +} + // Add fakes an Add event for obj. func (f *FakeInformer) Add(obj metav1.Object) { for _, h := range f.handlers { - h.OnAdd(obj) + h.OnAdd(obj, false) } } @@ -87,9 +105,9 @@ func (f *FakeInformer) Delete(obj metav1.Object) { } } -// AddEventHandlerWithResyncPeriod does nothing. TODO(community): Implement this. -func (f *FakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) { - +// RemoveEventHandler does nothing. TODO(community): Implement this. +func (f *FakeInformer) RemoveEventHandler(handle cache.ResourceEventHandlerRegistration) error { + return nil } // GetStore does nothing. TODO(community): Implement this. @@ -111,3 +129,18 @@ func (f *FakeInformer) LastSyncResourceVersion() string { func (f *FakeInformer) SetWatchErrorHandler(cache.WatchErrorHandler) error { return nil } + +// SetWatchErrorHandlerWithContext does nothing. TODO(community): Implement this. +func (f *FakeInformer) SetWatchErrorHandlerWithContext(cache.WatchErrorHandlerWithContext) error { + return nil +} + +// SetTransform does nothing. TODO(community): Implement this. +func (f *FakeInformer) SetTransform(t cache.TransformFunc) error { + return nil +} + +// IsStopped does nothing. TODO(community): Implement this. +func (f *FakeInformer) IsStopped() bool { + return false +} diff --git a/pkg/controller/controllerutil/controllerutil.go b/pkg/controller/controllerutil/controllerutil.go index 13f14a7ed3..0f12b934ee 100644 --- a/pkg/controller/controllerutil/controllerutil.go +++ b/pkg/controller/controllerutil/controllerutil.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "slices" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,7 +28,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) @@ -51,12 +53,22 @@ func newAlreadyOwnedError(obj metav1.Object, owner metav1.OwnerReference) *Alrea } } +// OwnerReferenceOption is a function that can modify a `metav1.OwnerReference`. +type OwnerReferenceOption func(*metav1.OwnerReference) + +// WithBlockOwnerDeletion allows configuring the BlockOwnerDeletion field on the `metav1.OwnerReference`. +func WithBlockOwnerDeletion(blockOwnerDeletion bool) OwnerReferenceOption { + return func(ref *metav1.OwnerReference) { + ref.BlockOwnerDeletion = &blockOwnerDeletion + } +} + // SetControllerReference sets owner as a Controller OwnerReference on controlled. // This is used for garbage collection of the controlled object and for // reconciling the owner object on changes to controlled (with a Watch + EnqueueRequestForOwner). // Since only one OwnerReference can be a controller, it returns an error if // there is another OwnerReference with Controller flag set. -func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme) error { +func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme, opts ...OwnerReferenceOption) error { // Validate the owner. ro, ok := owner.(runtime.Object) if !ok { @@ -76,8 +88,11 @@ func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Sch Kind: gvk.Kind, Name: owner.GetName(), UID: owner.GetUID(), - BlockOwnerDeletion: pointer.BoolPtr(true), - Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: ptr.To(true), + Controller: ptr.To(true), + } + for _, opt := range opts { + opt(&ref) } // Return early with an error if the object is already controlled. @@ -93,7 +108,7 @@ func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Sch // SetOwnerReference is a helper method to make sure the given object contains an object reference to the object provided. // This allows you to declare that owner has a dependency on the object without specifying it as a controller. // If a reference to the same object already exists, it'll be overwritten with the newly provided version. -func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) error { +func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme, opts ...OwnerReferenceOption) error { // Validate the owner. ro, ok := owner.(runtime.Object) if !ok { @@ -114,12 +129,108 @@ func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) erro UID: owner.GetUID(), Name: owner.GetName(), } + for _, opt := range opts { + opt(&ref) + } // Update owner references and return. upsertOwnerRef(ref, object) return nil } +// RemoveOwnerReference is a helper method to make sure the given object removes an owner reference to the object provided. +// This allows you to remove the owner to establish a new owner of the object in a subsequent call. +func RemoveOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) error { + owners := object.GetOwnerReferences() + length := len(owners) + if length < 1 { + return fmt.Errorf("%T does not have any owner references", object) + } + ro, ok := owner.(runtime.Object) + if !ok { + return fmt.Errorf("%T is not a runtime.Object, cannot call RemoveOwnerReference", owner) + } + gvk, err := apiutil.GVKForObject(ro, scheme) + if err != nil { + return err + } + + index := indexOwnerRef(owners, metav1.OwnerReference{ + APIVersion: gvk.GroupVersion().String(), + Name: owner.GetName(), + Kind: gvk.Kind, + }) + if index == -1 { + return fmt.Errorf("%T does not have an owner reference for %T", object, owner) + } + + owners = append(owners[:index], owners[index+1:]...) + object.SetOwnerReferences(owners) + return nil +} + +// HasControllerReference returns true if the object +// has an owner ref with controller equal to true +func HasControllerReference(object metav1.Object) bool { + owners := object.GetOwnerReferences() + for _, owner := range owners { + isTrue := owner.Controller + if owner.Controller != nil && *isTrue { + return true + } + } + return false +} + +// HasOwnerReference returns true if the owners list contains an owner reference +// that matches the object's group, kind, and name. +func HasOwnerReference(ownerRefs []metav1.OwnerReference, obj client.Object, scheme *runtime.Scheme) (bool, error) { + gvk, err := apiutil.GVKForObject(obj, scheme) + if err != nil { + return false, err + } + idx := indexOwnerRef(ownerRefs, metav1.OwnerReference{ + APIVersion: gvk.GroupVersion().String(), + Name: obj.GetName(), + Kind: gvk.Kind, + }) + return idx != -1, nil +} + +// RemoveControllerReference removes an owner reference where the controller +// equals true +func RemoveControllerReference(owner, object metav1.Object, scheme *runtime.Scheme) error { + if ok := HasControllerReference(object); !ok { + return fmt.Errorf("%T does not have a owner reference with controller equals true", object) + } + ro, ok := owner.(runtime.Object) + if !ok { + return fmt.Errorf("%T is not a runtime.Object, cannot call RemoveControllerReference", owner) + } + gvk, err := apiutil.GVKForObject(ro, scheme) + if err != nil { + return err + } + ownerRefs := object.GetOwnerReferences() + index := indexOwnerRef(ownerRefs, metav1.OwnerReference{ + APIVersion: gvk.GroupVersion().String(), + Name: owner.GetName(), + Kind: gvk.Kind, + }) + + if index == -1 { + return fmt.Errorf("%T does not have an controller reference for %T", object, owner) + } + + if ownerRefs[index].Controller == nil || !*ownerRefs[index].Controller { + return fmt.Errorf("%T owner is not the controller reference for %T", owner, object) + } + + ownerRefs = append(ownerRefs[:index], ownerRefs[index+1:]...) + object.SetOwnerReferences(ownerRefs) + return nil +} + func upsertOwnerRef(ref metav1.OwnerReference, object metav1.Object) { owners := object.GetOwnerReferences() if idx := indexOwnerRef(owners, ref); idx == -1 { @@ -165,11 +276,10 @@ func referSameObject(a, b metav1.OwnerReference) bool { if err != nil { return false } - return aGV.Group == bGV.Group && a.Kind == b.Kind && a.Name == b.Name } -// OperationResult is the action result of a CreateOrUpdate call. +// OperationResult is the action result of a CreateOrUpdate or CreateOrPatch call. type OperationResult string const ( // They should complete the sentence "Deployment default/foo has been ..." @@ -185,31 +295,52 @@ const ( // They should complete the sentence "Deployment default/foo has been .. OperationResultUpdatedStatusOnly OperationResult = "updatedStatusOnly" ) -// CreateOrUpdate creates or updates the given object in the Kubernetes -// cluster. The object's desired state must be reconciled with the existing -// state inside the passed in callback MutateFn. +// CreateOrUpdate attempts to fetch the given object from the Kubernetes cluster. +// If the object didn't exist, MutateFn will be called, and it will be created. +// If the object did exist, MutateFn will be called, and if it changed the +// object, it will be updated. +// Otherwise, it will be left unchanged. +// The executed operation (and an error) will be returned. +// +// WARNING: If the MutateFn resets a value on obj that has a default value, +// CreateOrUpdate will *always* perform an update. This is because when the +// object is fetched from the API server, the value will have taken on the +// default value, and the check for equality will fail. For example, Deployments +// must have a Replicas value set. If the MutateFn sets a Deployment's Replicas +// to nil, then it will never match with the object returned from the API +// server, which defaults the value to 1. // -// The MutateFn is called regardless of creating or updating an object. +// WARNING: CreateOrUpdate assumes that no values have been set on obj aside +// from the Name/Namespace. Values other than Name and Namespace that existed on +// obj may be overwritten by the corresponding values in the object returned +// from the Kubernetes API server. When this happens, the Update will not work +// as expected. // -// It returns the executed operation and an error. +// Note: changes made by MutateFn to any sub-resource (status...), will be +// discarded. func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (OperationResult, error) { key := client.ObjectKeyFromObject(obj) if err := c.Get(ctx, key, obj); err != nil { if !apierrors.IsNotFound(err) { return OperationResultNone, err } - if err := mutate(f, key, obj); err != nil { - return OperationResultNone, err + if f != nil { + if err := mutate(f, key, obj); err != nil { + return OperationResultNone, err + } } + if err := c.Create(ctx, obj); err != nil { return OperationResultNone, err } return OperationResultCreated, nil } - existing := obj.DeepCopyObject() //nolint - if err := mutate(f, key, obj); err != nil { - return OperationResultNone, err + existing := obj.DeepCopyObject() + if f != nil { + if err := mutate(f, key, obj); err != nil { + return OperationResultNone, err + } } if equality.Semantic.DeepEqual(existing, obj) { @@ -222,13 +353,32 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f M return OperationResultUpdated, nil } -// CreateOrPatch creates or patches the given object in the Kubernetes -// cluster. The object's desired state must be reconciled with the before -// state inside the passed in callback MutateFn. +// CreateOrPatch attempts to fetch the given object from the Kubernetes cluster. +// If the object didn't exist, MutateFn will be called, and it will be created. +// If the object did exist, MutateFn will be called, and if it changed the +// object, it will be patched. +// Otherwise, it will be left unchanged. +// The executed operation (and an error) will be returned. // -// The MutateFn is called regardless of creating or updating an object. +// WARNING: If the MutateFn resets a value on obj that has a default value, +// CreateOrPatch will *always* perform a patch. This is because when the +// object is fetched from the API server, the value will have taken on the +// default value, and the check for equality will fail. +// For example, Deployments must have a Replicas value set. If the MutateFn sets +// a Deployment's Replicas to nil, then it will never match with the object +// returned from the API server, which defaults the value to 1. // -// It returns the executed operation and an error. +// WARNING: CreateOrPatch assumes that no values have been set on obj aside +// from the Name/Namespace. Values other than Name and Namespace that existed on +// obj may be overwritten by the corresponding values in the object returned +// from the Kubernetes API server. When this happens, the Patch will not work +// as expected. +// +// Note: changes to any sub-resource other than status will be ignored. +// Changes to the status sub-resource will only be applied if the object +// already exist. To change the status on object creation, the easiest +// way is to requeue the object in the controller if OperationResult is +// OperationResultCreated func CreateOrPatch(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (OperationResult, error) { key := client.ObjectKeyFromObject(obj) if err := c.Get(ctx, key, obj); err != nil { @@ -345,45 +495,40 @@ func mutate(f MutateFn, key client.ObjectKey, obj client.Object) error { return nil } -// MutateFn is a function which mutates the existing object into it's desired state. +// MutateFn is a function which mutates the existing object into its desired state. type MutateFn func() error // AddFinalizer accepts an Object and adds the provided finalizer if not present. -func AddFinalizer(o client.Object, finalizer string) { +// It returns an indication of whether it updated the object's list of finalizers. +func AddFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) { f := o.GetFinalizers() - for _, e := range f { - if e == finalizer { - return - } + if slices.Contains(f, finalizer) { + return false } o.SetFinalizers(append(f, finalizer)) + return true } // RemoveFinalizer accepts an Object and removes the provided finalizer if present. -func RemoveFinalizer(o client.Object, finalizer string) { +// It returns an indication of whether it updated the object's list of finalizers. +func RemoveFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) { f := o.GetFinalizers() - for i := 0; i < len(f); i++ { + length := len(f) + + index := 0 + for i := range length { if f[i] == finalizer { - f = append(f[:i], f[i+1:]...) - i-- + continue } + f[index] = f[i] + index++ } - o.SetFinalizers(f) + o.SetFinalizers(f[:index]) + return length != index } // ContainsFinalizer checks an Object that the provided finalizer is present. func ContainsFinalizer(o client.Object, finalizer string) bool { f := o.GetFinalizers() - for _, e := range f { - if e == finalizer { - return true - } - } - return false + return slices.Contains(f, finalizer) } - -// Object allows functions to work indistinctly with any resource that -// implements both Object interfaces. -// -// Deprecated: Use client.Object instead. -type Object = client.Object diff --git a/pkg/controller/controllerutil/controllerutil_suite_test.go b/pkg/controller/controllerutil/controllerutil_suite_test.go index da4c5cf4ac..a4ac5cc746 100644 --- a/pkg/controller/controllerutil/controllerutil_suite_test.go +++ b/pkg/controller/controllerutil/controllerutil_suite_test.go @@ -19,19 +19,17 @@ package controllerutil_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestControllerutil(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Controllerutil Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Controllerutil Suite") } var testenv *envtest.Environment diff --git a/pkg/controller/controllerutil/controllerutil_test.go b/pkg/controller/controllerutil/controllerutil_test.go index d47c691465..a716667f6a 100644 --- a/pkg/controller/controllerutil/controllerutil_test.go +++ b/pkg/controller/controllerutil/controllerutil_test.go @@ -21,7 +21,7 @@ import ( "fmt" "math/rand" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -30,7 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -51,6 +51,23 @@ var _ = Describe("Controllerutil", func() { })) }) + It("should set the BlockOwnerDeletion if it is specified as an option", func() { + t := true + rs := &appsv1.ReplicaSet{} + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme, controllerutil.WithBlockOwnerDeletion(true))).ToNot(HaveOccurred()) + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid", + BlockOwnerDeletion: &t, + })) + }) + It("should not duplicate owner references", func() { rs := &appsv1.ReplicaSet{ ObjectMeta: metav1.ObjectMeta{ @@ -101,7 +118,161 @@ var _ = Describe("Controllerutil", func() { APIVersion: "extensions/v1beta1", UID: "foo-uid-2", })) + }) + It("should remove the owner reference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + { + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1alpha1", + UID: "foo-uid-1", + }, + }, + }, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid-2", + })) + Expect(controllerutil.RemoveOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) + }) + It("should remove the owner reference established by the SetControllerReference function", func() { + rs := &appsv1.ReplicaSet{} + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).NotTo(HaveOccurred()) + t := true + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid", + Controller: &t, + BlockOwnerDeletion: &t, + })) + Expect(controllerutil.RemoveOwnerReference(dep, rs, scheme.Scheme)).NotTo(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) + }) + It("should error when trying to remove the reference that doesn't exist", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.RemoveOwnerReference(dep, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) + }) + It("should error when trying to remove the reference that doesn't abide by the scheme", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveOwnerReference(dep, rs, runtime.NewScheme())).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(1)) + }) + It("should error when trying to remove the owner when setting the owner as a non runtime.Object", func() { + var obj metav1.Object + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveOwnerReference(obj, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(1)) + }) + + It("should error when trying to remove an owner that doesn't exist", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + dep2 := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "bar", UID: "bar-uid-3"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveOwnerReference(dep2, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(1)) + }) + It("should return true when HasControllerReference evaluates owner reference set by SetControllerReference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.HasControllerReference(rs)).To(BeTrue()) + }) + + It("should return false when HasControllerReference evaluates owner reference set by SetOwnerReference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.HasControllerReference(rs)).To(BeFalse()) + }) + + It("should error when RemoveControllerReference owner's controller is set to false", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveControllerReference(dep, rs, scheme.Scheme)).To(HaveOccurred()) + }) + + It("should error when RemoveControllerReference passed in owner is not the owner", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + dep2 := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo-2", UID: "foo-uid-42"}, + } + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.SetOwnerReference(dep2, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveControllerReference(dep2, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(2)) + }) + + It("should not error when RemoveControllerReference owner's controller is set to true", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) }) }) @@ -256,6 +427,25 @@ var _ = Describe("Controllerutil", func() { BlockOwnerDeletion: &t, })) }) + + It("should set the BlockOwnerDeletion if it is specified as an option", func() { + f := false + t := true + rs := &appsv1.ReplicaSet{} + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme, controllerutil.WithBlockOwnerDeletion(false))).NotTo(HaveOccurred()) + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid", + Controller: &t, + BlockOwnerDeletion: &f, + })) + }) }) Describe("CreateOrUpdate", func() { @@ -267,7 +457,7 @@ var _ = Describe("Controllerutil", func() { BeforeEach(func() { deploy = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("deploy-%d", rand.Int31()), //nolint:gosec + Name: fmt.Sprintf("deploy-%d", rand.Int31()), Namespace: "default", }, } @@ -301,8 +491,8 @@ var _ = Describe("Controllerutil", func() { specr = deploymentSpecr(deploy, deplSpec) }) - It("creates a new object if one doesn't exists", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("creates a new object if one doesn't exists", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -312,7 +502,7 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment created") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) By("being mutated by MutateFn") Expect(fetched.Spec.Template.Spec.Containers).To(HaveLen(1)) @@ -320,13 +510,13 @@ var _ = Describe("Controllerutil", func() { Expect(fetched.Spec.Template.Spec.Containers[0].Image).To(Equal(deplSpec.Template.Spec.Containers[0].Image)) }) - It("updates existing object", func() { + It("updates existing object", func(ctx SpecContext) { var scale int32 = 2 - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(err).NotTo(HaveOccurred()) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentScaler(deploy, scale)) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentScaler(deploy, scale)) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -335,17 +525,17 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment scaled") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) Expect(*fetched.Spec.Replicas).To(Equal(scale)) }) - It("updates only changed objects", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("updates only changed objects", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentIdentity) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentIdentity) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -353,8 +543,8 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when MutateFn changes object name on creation", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, func() error { + It("errors when MutateFn changes object name on creation", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, func() error { Expect(specr()).To(Succeed()) return deploymentRenamer(deploy)() }) @@ -366,13 +556,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when MutateFn renames an object", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("errors when MutateFn renames an object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentRenamer(deploy)) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentRenamer(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -381,13 +571,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when object namespace changes", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("errors when object namespace changes", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentNamespaceChanger(deploy)) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentNamespaceChanger(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -396,8 +586,8 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("aborts immediately if there was an error initially retrieving the object", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), errorReader{c}, deploy, func() error { + It("aborts immediately if there was an error initially retrieving the object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, errorReader{c}, deploy, func() error { Fail("Mutation method should not run") return nil }) @@ -416,7 +606,7 @@ var _ = Describe("Controllerutil", func() { BeforeEach(func() { deploy = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("deploy-%d", rand.Int31()), //nolint:gosec + Name: fmt.Sprintf("deploy-%d", rand.Int31()), Namespace: "default", }, } @@ -450,22 +640,22 @@ var _ = Describe("Controllerutil", func() { specr = deploymentSpecr(deploy, deplSpec) }) - assertLocalDeployWasUpdated := func(fetched *appsv1.Deployment) { + assertLocalDeployWasUpdated := func(ctx context.Context, fetched *appsv1.Deployment) { By("local deploy object was updated during patch & has same spec, status, resource version as fetched") if fetched == nil { fetched = &appsv1.Deployment{} - ExpectWithOffset(1, c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + ExpectWithOffset(1, c.Get(ctx, deplKey, fetched)).To(Succeed()) } ExpectWithOffset(1, fetched.ResourceVersion).To(Equal(deploy.ResourceVersion)) ExpectWithOffset(1, fetched.Spec).To(BeEquivalentTo(deploy.Spec)) ExpectWithOffset(1, fetched.Status).To(BeEquivalentTo(deploy.Status)) } - assertLocalDeployStatusWasUpdated := func(fetched *appsv1.Deployment) { + assertLocalDeployStatusWasUpdated := func(ctx context.Context, fetched *appsv1.Deployment) { By("local deploy object was updated during patch & has same spec, status, resource version as fetched") if fetched == nil { fetched = &appsv1.Deployment{} - ExpectWithOffset(1, c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + ExpectWithOffset(1, c.Get(ctx, deplKey, fetched)).To(Succeed()) } ExpectWithOffset(1, fetched.ResourceVersion).To(Equal(deploy.ResourceVersion)) ExpectWithOffset(1, *fetched.Spec.Replicas).To(BeEquivalentTo(int32(5))) @@ -473,8 +663,8 @@ var _ = Describe("Controllerutil", func() { ExpectWithOffset(1, len(fetched.Status.Conditions)).To(BeEquivalentTo(1)) } - It("creates a new object if one doesn't exists", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("creates a new object if one doesn't exists", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -484,7 +674,7 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment created") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) By("being mutated by MutateFn") Expect(fetched.Spec.Template.Spec.Containers).To(HaveLen(1)) @@ -492,13 +682,13 @@ var _ = Describe("Controllerutil", func() { Expect(fetched.Spec.Template.Spec.Containers[0].Image).To(Equal(deplSpec.Template.Spec.Containers[0].Image)) }) - It("patches existing object", func() { + It("patches existing object", func(ctx SpecContext) { var scale int32 = 2 - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(err).NotTo(HaveOccurred()) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentScaler(deploy, scale)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentScaler(deploy, scale)) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -507,29 +697,29 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment scaled") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) Expect(*fetched.Spec.Replicas).To(Equal(scale)) - assertLocalDeployWasUpdated(fetched) + assertLocalDeployWasUpdated(ctx, fetched) }) - It("patches only changed objects", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches only changed objects", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentIdentity) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentIdentity) By("returning no error") Expect(err).NotTo(HaveOccurred()) By("returning OperationResultNone") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) }) - It("patches only changed status", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches only changed status", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) @@ -538,18 +728,18 @@ var _ = Describe("Controllerutil", func() { ReadyReplicas: 1, Replicas: 3, } - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentStatusr(deploy, deployStatus)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentStatusr(deploy, deployStatus)) By("returning no error") Expect(err).NotTo(HaveOccurred()) By("returning OperationResultUpdatedStatusOnly") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatusOnly)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) }) - It("patches resource and status", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches resource and status", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) @@ -559,7 +749,7 @@ var _ = Describe("Controllerutil", func() { ReadyReplicas: 1, Replicas: replicas, } - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, func() error { Expect(deploymentScaler(deploy, replicas)()).To(Succeed()) return deploymentStatusr(deploy, deployStatus)() }) @@ -569,11 +759,11 @@ var _ = Describe("Controllerutil", func() { By("returning OperationResultUpdatedStatus") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatus)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) }) - It("patches resource and not empty status", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches resource and not empty status", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) @@ -583,7 +773,7 @@ var _ = Describe("Controllerutil", func() { ReadyReplicas: 1, Replicas: replicas, } - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, func() error { Expect(deploymentScaler(deploy, replicas)()).To(Succeed()) return deploymentStatusr(deploy, deployStatus)() }) @@ -593,10 +783,10 @@ var _ = Describe("Controllerutil", func() { By("returning OperationResultUpdatedStatus") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatus)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { - deploy.Spec.Replicas = pointer.Int32Ptr(5) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, func() error { + deploy.Spec.Replicas = ptr.To(int32(5)) deploy.Status.Conditions = []appsv1.DeploymentCondition{{ Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, @@ -609,11 +799,11 @@ var _ = Describe("Controllerutil", func() { By("returning OperationResultUpdatedStatus") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatus)) - assertLocalDeployStatusWasUpdated(nil) + assertLocalDeployStatusWasUpdated(ctx, nil) }) - It("errors when MutateFn changes object name on creation", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { + It("errors when MutateFn changes object name on creation", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, func() error { Expect(specr()).To(Succeed()) return deploymentRenamer(deploy)() }) @@ -625,13 +815,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when MutateFn renames an object", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("errors when MutateFn renames an object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentRenamer(deploy)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentRenamer(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -640,13 +830,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when object namespace changes", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("errors when object namespace changes", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentNamespaceChanger(deploy)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentNamespaceChanger(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -655,8 +845,8 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("aborts immediately if there was an error initially retrieving the object", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), errorReader{c}, deploy, func() error { + It("aborts immediately if there was an error initially retrieving the object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, errorReader{c}, deploy, func() error { Fail("Mutation method should not run") return nil }) @@ -700,24 +890,112 @@ var _ = Describe("Controllerutil", func() { }) }) + Describe("AddFinalizer, which returns an indication of whether it modified the object's list of finalizers,", func() { + deploy = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{}, + }, + } + + When("the object's list of finalizers has no instances of the input finalizer", func() { + It("should return true", func() { + Expect(controllerutil.AddFinalizer(deploy, testFinalizer)).To(BeTrue()) + }) + It("should add the input finalizer to the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{testFinalizer})) + }) + }) + + When("the object's list of finalizers has an instance of the input finalizer", func() { + It("should return false", func() { + Expect(controllerutil.AddFinalizer(deploy, testFinalizer)).To(BeFalse()) + }) + It("should not modify the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{testFinalizer})) + }) + }) + }) + + Describe("RemoveFinalizer, which returns an indication of whether it modified the object's list of finalizers,", func() { + When("the object's list of finalizers has no instances of the input finalizer", func() { + It("should return false", func() { + Expect(controllerutil.RemoveFinalizer(deploy, testFinalizer1)).To(BeFalse()) + }) + It("should not modify the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{testFinalizer})) + }) + }) + + When("the object's list of finalizers has one instance of the input finalizer", func() { + It("should return true", func() { + Expect(controllerutil.RemoveFinalizer(deploy, testFinalizer)).To(BeTrue()) + }) + It("should remove the instance of the input finalizer from the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{})) + }) + }) + + When("the object's list of finalizers has multiple instances of the input finalizer", func() { + It("should return true", func() { + deploy.SetFinalizers(append(deploy.Finalizers, testFinalizer, testFinalizer)) + Expect(controllerutil.RemoveFinalizer(deploy, testFinalizer)).To(BeTrue()) + }) + It("should remove each instance of the input finalizer from the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{})) + }) + }) + }) + Describe("ContainsFinalizer", func() { It("should check that finalizer is present", func() { controllerutil.AddFinalizer(deploy, testFinalizer) - Expect(controllerutil.ContainsFinalizer(deploy, testFinalizer)).To(Equal(true)) + Expect(controllerutil.ContainsFinalizer(deploy, testFinalizer)).To(BeTrue()) }) It("should check that finalizer is not present after RemoveFinalizer call", func() { controllerutil.RemoveFinalizer(deploy, testFinalizer) - Expect(controllerutil.ContainsFinalizer(deploy, testFinalizer)).To(Equal(false)) + Expect(controllerutil.ContainsFinalizer(deploy, testFinalizer)).To(BeFalse()) + }) + }) + + Describe("HasOwnerReference", func() { + It("should return true if the object has the owner reference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + b, err := controllerutil.HasOwnerReference(rs.GetOwnerReferences(), dep, scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + Expect(b).To(BeTrue()) + }) + + It("should return false if the object does not have the owner reference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + b, err := controllerutil.HasOwnerReference(rs.GetOwnerReferences(), dep, scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + Expect(b).To(BeFalse()) }) }) }) }) -const testFinalizer = "foo.bar.baz" +const ( + testFinalizer = "foo.bar.baz" + testFinalizer1 = testFinalizer + "1" +) -var _ runtime.Object = &errRuntimeObj{} -var _ metav1.Object = &errMetaObj{} +var ( + _ runtime.Object = &errRuntimeObj{} + _ metav1.Object = &errMetaObj{} +) type errRuntimeObj struct { runtime.TypeMeta @@ -775,6 +1053,6 @@ type errorReader struct { client.Client } -func (e errorReader) Get(ctx context.Context, key client.ObjectKey, into client.Object) error { +func (e errorReader) Get(ctx context.Context, key client.ObjectKey, into client.Object, opts ...client.GetOption) error { return fmt.Errorf("unexpected error") } diff --git a/pkg/controller/doc.go b/pkg/controller/doc.go index 667b14fdd7..228335e929 100644 --- a/pkg/controller/doc.go +++ b/pkg/controller/doc.go @@ -17,7 +17,7 @@ limitations under the License. /* Package controller provides types and functions for building Controllers. Controllers implement Kubernetes APIs. -Creation +# Creation To create a new Controller, first create a manager.Manager and pass it to the controller.New function. The Controller MUST be started by calling Manager.Start. diff --git a/pkg/controller/example_test.go b/pkg/controller/example_test.go index 3d8e399703..e3c4b6a092 100644 --- a/pkg/controller/example_test.go +++ b/pkg/controller/example_test.go @@ -71,7 +71,7 @@ func ExampleController() { } // Watch for Pod create / update / delete events and call Reconcile - err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{})) if err != nil { log.Error(err, "unable to watch pods") os.Exit(1) @@ -108,7 +108,7 @@ func ExampleController_unstructured() { Version: "v1", }) // Watch for Pod create / update / delete events and call Reconcile - err = c.Watch(&source.Kind{Type: u}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), u, &handler.TypedEnqueueRequestForObject[*unstructured.Unstructured]{})) if err != nil { log.Error(err, "unable to watch pods") os.Exit(1) @@ -129,7 +129,7 @@ func ExampleNewUnmanaged() { // Configure creates a new controller but does not add it to the supplied // manager. - c, err := controller.NewUnmanaged("pod-controller", mgr, controller.Options{ + c, err := controller.NewUnmanaged("pod-controller", controller.Options{ Reconciler: reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { return reconcile.Result{}, nil }), @@ -139,7 +139,7 @@ func ExampleNewUnmanaged() { os.Exit(1) } - if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{})); err != nil { log.Error(err, "unable to watch pods") os.Exit(1) } diff --git a/pkg/ratelimiter/ratelimiter.go b/pkg/controller/name.go similarity index 50% rename from pkg/ratelimiter/ratelimiter.go rename to pkg/controller/name.go index 565a3a227f..00ca655128 100644 --- a/pkg/ratelimiter/ratelimiter.go +++ b/pkg/controller/name.go @@ -14,17 +14,30 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ratelimiter - -import "time" - -// RateLimiter is an identical interface of client-go workqueue RateLimiter. -type RateLimiter interface { - // When gets an item and gets to decide how long that item should wait - When(item interface{}) time.Duration - // Forget indicates that an item is finished being retried. Doesn't matter whether its for perm failing - // or for success, we'll stop tracking it - Forget(item interface{}) - // NumRequeues returns back how many failures the item has had - NumRequeues(item interface{}) int +package controller + +import ( + "fmt" + "sync" + + "k8s.io/apimachinery/pkg/util/sets" +) + +var nameLock sync.Mutex +var usedNames sets.Set[string] + +func checkName(name string) error { + nameLock.Lock() + defer nameLock.Unlock() + if usedNames == nil { + usedNames = sets.Set[string]{} + } + + if usedNames.Has(name) { + return fmt.Errorf("controller with name %s already exists. Controller names must be unique to avoid multiple controllers reporting the same metric. This validation can be disabled via the SkipNameValidation option", name) + } + + usedNames.Insert(name) + + return nil } diff --git a/pkg/controller/priorityqueue/metrics.go b/pkg/controller/priorityqueue/metrics.go new file mode 100644 index 0000000000..967a252dfb --- /dev/null +++ b/pkg/controller/priorityqueue/metrics.go @@ -0,0 +1,172 @@ +package priorityqueue + +import ( + "sync" + "time" + + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" + "sigs.k8s.io/controller-runtime/pkg/internal/metrics" +) + +// This file is mostly a copy of unexported code from +// https://github.com/kubernetes/kubernetes/blob/1d8828ce707ed9dd7a6a9756385419cce1d202ac/staging/src/k8s.io/client-go/util/workqueue/metrics.go +// +// The only two differences are the addition of mapLock in defaultQueueMetrics and converging retryMetrics into queueMetrics. + +type queueMetrics[T comparable] interface { + add(item T, priority int) + get(item T, priority int) + updateDepthWithPriorityMetric(oldPriority, newPriority int) + done(item T) + updateUnfinishedWork() + retry() +} + +func newQueueMetrics[T comparable](mp workqueue.MetricsProvider, name string, clock clock.Clock) queueMetrics[T] { + if len(name) == 0 { + return noMetrics[T]{} + } + + dqm := &defaultQueueMetrics[T]{ + clock: clock, + adds: mp.NewAddsMetric(name), + latency: mp.NewLatencyMetric(name), + workDuration: mp.NewWorkDurationMetric(name), + unfinishedWorkSeconds: mp.NewUnfinishedWorkSecondsMetric(name), + longestRunningProcessor: mp.NewLongestRunningProcessorSecondsMetric(name), + addTimes: map[T]time.Time{}, + processingStartTimes: map[T]time.Time{}, + retries: mp.NewRetriesMetric(name), + } + + if mpp, ok := mp.(metrics.MetricsProviderWithPriority); ok { + dqm.depthWithPriority = mpp.NewDepthMetricWithPriority(name) + } else { + dqm.depth = mp.NewDepthMetric(name) + } + return dqm +} + +// defaultQueueMetrics expects the caller to lock before setting any metrics. +type defaultQueueMetrics[T comparable] struct { + clock clock.Clock + + // current depth of a workqueue + depth workqueue.GaugeMetric + depthWithPriority metrics.DepthMetricWithPriority + // total number of adds handled by a workqueue + adds workqueue.CounterMetric + // how long an item stays in a workqueue + latency workqueue.HistogramMetric + // how long processing an item from a workqueue takes + workDuration workqueue.HistogramMetric + + mapLock sync.RWMutex + addTimes map[T]time.Time + processingStartTimes map[T]time.Time + + // how long have current threads been working? + unfinishedWorkSeconds workqueue.SettableGaugeMetric + longestRunningProcessor workqueue.SettableGaugeMetric + + retries workqueue.CounterMetric +} + +// add is called for ready items only +func (m *defaultQueueMetrics[T]) add(item T, priority int) { + if m == nil { + return + } + + m.adds.Inc() + if m.depthWithPriority != nil { + m.depthWithPriority.Inc(priority) + } else { + m.depth.Inc() + } + + m.mapLock.Lock() + defer m.mapLock.Unlock() + + if _, exists := m.addTimes[item]; !exists { + m.addTimes[item] = m.clock.Now() + } +} + +func (m *defaultQueueMetrics[T]) get(item T, priority int) { + if m == nil { + return + } + + if m.depthWithPriority != nil { + m.depthWithPriority.Dec(priority) + } else { + m.depth.Dec() + } + + m.mapLock.Lock() + defer m.mapLock.Unlock() + + m.processingStartTimes[item] = m.clock.Now() + if startTime, exists := m.addTimes[item]; exists { + m.latency.Observe(m.sinceInSeconds(startTime)) + delete(m.addTimes, item) + } +} + +func (m *defaultQueueMetrics[T]) updateDepthWithPriorityMetric(oldPriority, newPriority int) { + if m.depthWithPriority != nil { + m.depthWithPriority.Dec(oldPriority) + m.depthWithPriority.Inc(newPriority) + } +} + +func (m *defaultQueueMetrics[T]) done(item T) { + if m == nil { + return + } + + m.mapLock.Lock() + defer m.mapLock.Unlock() + if startTime, exists := m.processingStartTimes[item]; exists { + m.workDuration.Observe(m.sinceInSeconds(startTime)) + delete(m.processingStartTimes, item) + } +} + +func (m *defaultQueueMetrics[T]) updateUnfinishedWork() { + m.mapLock.RLock() + defer m.mapLock.RUnlock() + // Note that a summary metric would be better for this, but prometheus + // doesn't seem to have non-hacky ways to reset the summary metrics. + var total float64 + var oldest float64 + for _, t := range m.processingStartTimes { + age := m.sinceInSeconds(t) + total += age + if age > oldest { + oldest = age + } + } + m.unfinishedWorkSeconds.Set(total) + m.longestRunningProcessor.Set(oldest) +} + +// Gets the time since the specified start in seconds. +func (m *defaultQueueMetrics[T]) sinceInSeconds(start time.Time) float64 { + return m.clock.Since(start).Seconds() +} + +func (m *defaultQueueMetrics[T]) retry() { + m.retries.Inc() +} + +type noMetrics[T any] struct{} + +func (noMetrics[T]) add(item T, priority int) {} +func (noMetrics[T]) get(item T, priority int) {} +func (noMetrics[T]) updateDepthWithPriorityMetric(oldPriority, newPriority int) {} +func (noMetrics[T]) done(item T) {} +func (noMetrics[T]) updateUnfinishedWork() {} +func (noMetrics[T]) retry() {} diff --git a/pkg/controller/priorityqueue/metrics_test.go b/pkg/controller/priorityqueue/metrics_test.go new file mode 100644 index 0000000000..3be3989d89 --- /dev/null +++ b/pkg/controller/priorityqueue/metrics_test.go @@ -0,0 +1,141 @@ +package priorityqueue + +import ( + "sync" + + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/internal/metrics" +) + +func newFakeMetricsProvider() *fakeMetricsProvider { + return &fakeMetricsProvider{ + depth: make(map[string]map[int]int), + adds: make(map[string]int), + latency: make(map[string][]float64), + workDuration: make(map[string][]float64), + unfinishedWorkSeconds: make(map[string]float64), + longestRunningProcessor: make(map[string]float64), + retries: make(map[string]int), + mu: sync.Mutex{}, + } +} + +var _ metrics.MetricsProviderWithPriority = &fakeMetricsProvider{} + +type fakeMetricsProvider struct { + depth map[string]map[int]int + adds map[string]int + latency map[string][]float64 + workDuration map[string][]float64 + unfinishedWorkSeconds map[string]float64 + longestRunningProcessor map[string]float64 + retries map[string]int + mu sync.Mutex +} + +func (f *fakeMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { + panic("Should never be called. Expected NewDepthMetricWithPriority to be called instead") +} + +func (f *fakeMetricsProvider) NewDepthMetricWithPriority(name string) metrics.DepthMetricWithPriority { + f.mu.Lock() + defer f.mu.Unlock() + f.depth[name] = map[int]int{} + return &fakeGaugeMetric{m: &f.depth, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.adds[name] = 0 + return &fakeCounterMetric{m: &f.adds, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.latency[name] = []float64{} + return &fakeHistogramMetric{m: &f.latency, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.workDuration[name] = []float64{} + return &fakeHistogramMetric{m: &f.workDuration, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.unfinishedWorkSeconds[name] = 0 + return &fakeSettableGaugeMetric{m: &f.unfinishedWorkSeconds, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.longestRunningProcessor[name] = 0 + return &fakeSettableGaugeMetric{m: &f.longestRunningProcessor, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.retries[name] = 0 + return &fakeCounterMetric{m: &f.retries, mu: &f.mu, name: name} +} + +type fakeGaugeMetric struct { + m *map[string]map[int]int + mu *sync.Mutex + name string +} + +func (fg *fakeGaugeMetric) Inc(priority int) { + fg.mu.Lock() + defer fg.mu.Unlock() + (*fg.m)[fg.name][priority]++ +} + +func (fg *fakeGaugeMetric) Dec(priority int) { + fg.mu.Lock() + defer fg.mu.Unlock() + (*fg.m)[fg.name][priority]-- +} + +type fakeCounterMetric struct { + m *map[string]int + mu *sync.Mutex + name string +} + +func (fc *fakeCounterMetric) Inc() { + fc.mu.Lock() + defer fc.mu.Unlock() + (*fc.m)[fc.name]++ +} + +type fakeHistogramMetric struct { + m *map[string][]float64 + mu *sync.Mutex + name string +} + +func (fh *fakeHistogramMetric) Observe(v float64) { + fh.mu.Lock() + defer fh.mu.Unlock() + (*fh.m)[fh.name] = append((*fh.m)[fh.name], v) +} + +type fakeSettableGaugeMetric struct { + m *map[string]float64 + mu *sync.Mutex + name string +} + +func (fs *fakeSettableGaugeMetric) Set(v float64) { + fs.mu.Lock() + defer fs.mu.Unlock() + (*fs.m)[fs.name] = v +} diff --git a/pkg/controller/priorityqueue/priorityqueue.go b/pkg/controller/priorityqueue/priorityqueue.go new file mode 100644 index 0000000000..98df84c56b --- /dev/null +++ b/pkg/controller/priorityqueue/priorityqueue.go @@ -0,0 +1,460 @@ +package priorityqueue + +import ( + "math" + "sync" + "sync/atomic" + "time" + + "github.com/go-logr/logr" + "github.com/google/btree" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/internal/metrics" +) + +// AddOpts describes the options for adding items to the queue. +type AddOpts struct { + After time.Duration + RateLimited bool + // Priority is the priority of the item. Higher values + // indicate higher priority. + // Defaults to zero if unset. + Priority *int +} + +// PriorityQueue is a priority queue for a controller. It +// internally de-duplicates all items that are added to +// it. It will use the max of the passed priorities and the +// min of possible durations. +type PriorityQueue[T comparable] interface { + workqueue.TypedRateLimitingInterface[T] + AddWithOpts(o AddOpts, Items ...T) + GetWithPriority() (item T, priority int, shutdown bool) +} + +// Opts contains the options for a PriorityQueue. +type Opts[T comparable] struct { + // Ratelimiter is being used when AddRateLimited is called. Defaults to a per-item exponential backoff + // limiter with an initial delay of five milliseconds and a max delay of 1000 seconds. + RateLimiter workqueue.TypedRateLimiter[T] + MetricProvider workqueue.MetricsProvider + Log logr.Logger +} + +// Opt allows to configure a PriorityQueue. +type Opt[T comparable] func(*Opts[T]) + +// New constructs a new PriorityQueue. +func New[T comparable](name string, o ...Opt[T]) PriorityQueue[T] { + opts := &Opts[T]{} + for _, f := range o { + f(opts) + } + + if opts.RateLimiter == nil { + opts.RateLimiter = workqueue.NewTypedItemExponentialFailureRateLimiter[T](5*time.Millisecond, 1000*time.Second) + } + + if opts.MetricProvider == nil { + opts.MetricProvider = metrics.WorkqueueMetricsProvider{} + } + + pq := &priorityqueue[T]{ + log: opts.Log, + items: map[T]*item[T]{}, + queue: btree.NewG(32, less[T]), + becameReady: sets.Set[T]{}, + metrics: newQueueMetrics[T](opts.MetricProvider, name, clock.RealClock{}), + // itemOrWaiterAdded indicates that an item or + // waiter was added. It must be buffered, because + // if we currently process items we can't tell + // if that included the new item/waiter. + itemOrWaiterAdded: make(chan struct{}, 1), + rateLimiter: opts.RateLimiter, + locked: sets.Set[T]{}, + done: make(chan struct{}), + get: make(chan item[T]), + now: time.Now, + tick: time.Tick, + } + + go pq.spin() + go pq.logState() + if _, ok := pq.metrics.(noMetrics[T]); !ok { + go pq.updateUnfinishedWorkLoop() + } + + return pq +} + +type priorityqueue[T comparable] struct { + log logr.Logger + // lock has to be acquired for any access any of items, queue, addedCounter + // or becameReady + lock sync.Mutex + items map[T]*item[T] + queue bTree[*item[T]] + + // addedCounter is a counter of elements added, we need it + // because unixNano is not guaranteed to be unique. + addedCounter uint64 + + // becameReady holds items that are in the queue, were added + // with non-zero after and became ready. We need it to call the + // metrics add exactly once for them. + becameReady sets.Set[T] + metrics queueMetrics[T] + + itemOrWaiterAdded chan struct{} + + rateLimiter workqueue.TypedRateLimiter[T] + + // locked contains the keys we handed out through Get() and that haven't + // yet been returned through Done(). + locked sets.Set[T] + lockedLock sync.RWMutex + + shutdown atomic.Bool + done chan struct{} + + get chan item[T] + + // waiters is the number of routines blocked in Get, we use it to determine + // if we can push items. + waiters atomic.Int64 + + // Configurable for testing + now func() time.Time + tick func(time.Duration) <-chan time.Time +} + +func (w *priorityqueue[T]) AddWithOpts(o AddOpts, items ...T) { + if w.shutdown.Load() { + return + } + + w.lock.Lock() + defer w.lock.Unlock() + + for _, key := range items { + after := o.After + if o.RateLimited { + rlAfter := w.rateLimiter.When(key) + if after == 0 || rlAfter < after { + after = rlAfter + } + } + + var readyAt *time.Time + if after > 0 { + readyAt = ptr.To(w.now().Add(after)) + w.metrics.retry() + } + if _, ok := w.items[key]; !ok { + item := &item[T]{ + Key: key, + AddedCounter: w.addedCounter, + Priority: ptr.Deref(o.Priority, 0), + ReadyAt: readyAt, + } + w.items[key] = item + w.queue.ReplaceOrInsert(item) + if item.ReadyAt == nil { + w.metrics.add(key, item.Priority) + } + w.addedCounter++ + continue + } + + // The b-tree de-duplicates based on ordering and any change here + // will affect the order - Just delete and re-add. + item, _ := w.queue.Delete(w.items[key]) + if newPriority := ptr.Deref(o.Priority, 0); newPriority > item.Priority { + // Update depth metric only if the item in the queue was already added to the depth metric. + if item.ReadyAt == nil || w.becameReady.Has(key) { + w.metrics.updateDepthWithPriorityMetric(item.Priority, newPriority) + } + item.Priority = newPriority + } + + if item.ReadyAt != nil && (readyAt == nil || readyAt.Before(*item.ReadyAt)) { + if readyAt == nil && !w.becameReady.Has(key) { + w.metrics.add(key, item.Priority) + } + item.ReadyAt = readyAt + } + + w.queue.ReplaceOrInsert(item) + } + + if len(items) > 0 { + w.notifyItemOrWaiterAdded() + } +} + +func (w *priorityqueue[T]) notifyItemOrWaiterAdded() { + select { + case w.itemOrWaiterAdded <- struct{}{}: + default: + } +} + +func (w *priorityqueue[T]) spin() { + blockForever := make(chan time.Time) + var nextReady <-chan time.Time + nextReady = blockForever + var nextItemReadyAt time.Time + + for { + select { + case <-w.done: + return + case <-w.itemOrWaiterAdded: + case <-nextReady: + nextReady = blockForever + nextItemReadyAt = time.Time{} + } + + func() { + w.lock.Lock() + defer w.lock.Unlock() + + w.lockedLock.Lock() + defer w.lockedLock.Unlock() + + // manipulating the tree from within Ascend might lead to panics, so + // track what we want to delete and do it after we are done ascending. + var toDelete []*item[T] + + var key T + + // Items in the queue tree are sorted first by priority and second by readiness, so + // items with a lower priority might be ready further down in the queue. + // We iterate through the priorities high to low until we find a ready item + pivot := item[T]{ + Key: key, + AddedCounter: 0, + Priority: math.MaxInt, + ReadyAt: nil, + } + + for { + pivotChange := false + + w.queue.AscendGreaterOrEqual(&pivot, func(item *item[T]) bool { + // Item is locked, we can not hand it out + if w.locked.Has(item.Key) { + return true + } + + if item.ReadyAt != nil { + if readyAt := item.ReadyAt.Sub(w.now()); readyAt > 0 { + if nextItemReadyAt.After(*item.ReadyAt) || nextItemReadyAt.IsZero() { + nextReady = w.tick(readyAt) + nextItemReadyAt = *item.ReadyAt + } + + // Adjusting the pivot item moves the ascend to the next lower priority + pivot.Priority = item.Priority - 1 + pivotChange = true + return false + } + if !w.becameReady.Has(item.Key) { + w.metrics.add(item.Key, item.Priority) + w.becameReady.Insert(item.Key) + } + } + + if w.waiters.Load() == 0 { + // Have to keep iterating here to ensure we update metrics + // for further items that became ready and set nextReady. + return true + } + + w.metrics.get(item.Key, item.Priority) + w.locked.Insert(item.Key) + w.waiters.Add(-1) + delete(w.items, item.Key) + toDelete = append(toDelete, item) + w.becameReady.Delete(item.Key) + w.get <- *item + + return true + }) + + if !pivotChange { + break + } + } + + for _, item := range toDelete { + w.queue.Delete(item) + } + }() + } +} + +func (w *priorityqueue[T]) Add(item T) { + w.AddWithOpts(AddOpts{}, item) +} + +func (w *priorityqueue[T]) AddAfter(item T, after time.Duration) { + w.AddWithOpts(AddOpts{After: after}, item) +} + +func (w *priorityqueue[T]) AddRateLimited(item T) { + w.AddWithOpts(AddOpts{RateLimited: true}, item) +} + +func (w *priorityqueue[T]) GetWithPriority() (_ T, priority int, shutdown bool) { + if w.shutdown.Load() { + var zero T + return zero, 0, true + } + + w.waiters.Add(1) + + w.notifyItemOrWaiterAdded() + + select { + case <-w.done: + // Return if the queue was shutdown while we were already waiting for an item here. + // For example controller workers are continuously calling GetWithPriority and + // GetWithPriority is blocking the workers if there are no items in the queue. + // If the controller and accordingly the queue is then shut down, without this code + // branch the controller workers remain blocked here and are unable to shut down. + var zero T + return zero, 0, true + case item := <-w.get: + return item.Key, item.Priority, w.shutdown.Load() + } +} + +func (w *priorityqueue[T]) Get() (item T, shutdown bool) { + key, _, shutdown := w.GetWithPriority() + return key, shutdown +} + +func (w *priorityqueue[T]) Forget(item T) { + w.rateLimiter.Forget(item) +} + +func (w *priorityqueue[T]) NumRequeues(item T) int { + return w.rateLimiter.NumRequeues(item) +} + +func (w *priorityqueue[T]) ShuttingDown() bool { + return w.shutdown.Load() +} + +func (w *priorityqueue[T]) Done(item T) { + w.lockedLock.Lock() + defer w.lockedLock.Unlock() + w.locked.Delete(item) + w.metrics.done(item) + w.notifyItemOrWaiterAdded() +} + +func (w *priorityqueue[T]) ShutDown() { + w.shutdown.Store(true) + close(w.done) +} + +// ShutDownWithDrain just calls ShutDown, as the draining +// functionality is not used by controller-runtime. +func (w *priorityqueue[T]) ShutDownWithDrain() { + w.ShutDown() +} + +// Len returns the number of items that are ready to be +// picked up. It does not include items that are not yet +// ready. +func (w *priorityqueue[T]) Len() int { + w.lock.Lock() + defer w.lock.Unlock() + + var result int + w.queue.Ascend(func(item *item[T]) bool { + if item.ReadyAt == nil || item.ReadyAt.Compare(w.now()) <= 0 { + result++ + return true + } + return false + }) + + return result +} + +func (w *priorityqueue[T]) logState() { + t := time.Tick(10 * time.Second) + for { + select { + case <-w.done: + return + case <-t: + } + + // Log level may change at runtime, so keep the + // loop going even if a given level is currently + // not enabled. + if !w.log.V(5).Enabled() { + continue + } + w.lock.Lock() + items := make([]*item[T], 0, len(w.items)) + w.queue.Ascend(func(item *item[T]) bool { + items = append(items, item) + return true + }) + w.lock.Unlock() + + w.log.V(5).Info("workqueue_items", "items", items) + } +} + +func less[T comparable](a, b *item[T]) bool { + if a.Priority != b.Priority { + return a.Priority > b.Priority + } + if a.ReadyAt == nil && b.ReadyAt != nil { + return true + } + if b.ReadyAt == nil && a.ReadyAt != nil { + return false + } + if a.ReadyAt != nil && b.ReadyAt != nil && !a.ReadyAt.Equal(*b.ReadyAt) { + return a.ReadyAt.Before(*b.ReadyAt) + } + + return a.AddedCounter < b.AddedCounter +} + +type item[T comparable] struct { + Key T `json:"key"` + AddedCounter uint64 `json:"addedCounter"` + Priority int `json:"priority"` + ReadyAt *time.Time `json:"readyAt,omitempty"` +} + +func (w *priorityqueue[T]) updateUnfinishedWorkLoop() { + t := time.Tick(500 * time.Millisecond) // borrowed from workqueue: https://github.com/kubernetes/kubernetes/blob/67a807bf142c7a2a5ecfdb2a5d24b4cdea4cc79c/staging/src/k8s.io/client-go/util/workqueue/queue.go#L182 + for { + select { + case <-w.done: + return + case <-t: + } + w.metrics.updateUnfinishedWork() + } +} + +type bTree[T any] interface { + ReplaceOrInsert(item T) (_ T, _ bool) + Delete(item T) (T, bool) + Ascend(iterator btree.ItemIteratorG[T]) + AscendGreaterOrEqual(pivot T, iterator btree.ItemIteratorG[T]) +} diff --git a/pkg/controller/priorityqueue/priorityqueue_suite_test.go b/pkg/controller/priorityqueue/priorityqueue_suite_test.go new file mode 100644 index 0000000000..71bc5ba049 --- /dev/null +++ b/pkg/controller/priorityqueue/priorityqueue_suite_test.go @@ -0,0 +1,13 @@ +package priorityqueue + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestControllerWorkqueue(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ControllerWorkqueue Suite") +} diff --git a/pkg/controller/priorityqueue/priorityqueue_test.go b/pkg/controller/priorityqueue/priorityqueue_test.go new file mode 100644 index 0000000000..e18a6393eb --- /dev/null +++ b/pkg/controller/priorityqueue/priorityqueue_test.go @@ -0,0 +1,775 @@ +package priorityqueue + +import ( + "fmt" + "math/rand/v2" + "sync" + "testing" + "time" + + fuzz "github.com/google/gofuzz" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" +) + +var _ = Describe("Controllerworkqueue", func() { + It("returns an item", func() { + q, metrics := newQueue() + defer q.ShutDown() + q.AddWithOpts(AddOpts{}, "foo") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + Expect(metrics.retries["test"]).To(Equal(0)) + }) + + It("returns items in order", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("bar")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + }) + + It("doesn't return an item that is currently locked", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("bar")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + Expect(metrics.adds["test"]).To(Equal(3)) + }) + + It("returns an item as soon as its unlocked", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("bar")) + + q.AddWithOpts(AddOpts{}, "baz") + q.Done("foo") + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("foo")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + Expect(metrics.adds["test"]).To(Equal(4)) + }) + + It("de-duplicates items", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "foo") + + Expect(q.Len()).To(Equal(1)) + + q.lockedLock.Lock() + Expect(q.locked.Len()).To(Equal(0)) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("retains the highest priority", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{Priority: ptr.To(1)}, "foo") + q.AddWithOpts(AddOpts{Priority: ptr.To(2)}, "foo") + + item, priority, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + Expect(priority).To(Equal(2)) + + Expect(q.Len()).To(Equal(0)) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{1: 0, 2: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("gets pushed to the front if the priority increases", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{Priority: ptr.To(1)}, "baz") + + item, priority, _ := q.GetWithPriority() + Expect(item).To(Equal("baz")) + Expect(priority).To(Equal(1)) + + Expect(q.Len()).To(Equal(2)) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2, 1: 0})) + Expect(metrics.adds["test"]).To(Equal(3)) + }) + + It("retains the lowest after duration", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: 0}, "foo") + q.AddWithOpts(AddOpts{After: time.Hour}, "foo") + + item, priority, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + Expect(priority).To(Equal(0)) + + Expect(q.Len()).To(Equal(0)) + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("returns an item only after after has passed", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + Expect(d).To(Equal(time.Second)) + return originalTick(d) + } + + retrievedItem := make(chan struct{}) + + go func() { + defer GinkgoRecover() + q.GetWithPriority() + close(retrievedItem) + }() + + q.AddWithOpts(AddOpts{After: time.Second}, "foo") + + Consistently(retrievedItem).ShouldNot(BeClosed()) + + forwardQueueTimeBy(time.Second) + Eventually(retrievedItem).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + Expect(metrics.retries["test"]).To(Equal(1)) + }) + + It("returns high priority item that became ready before low priority item", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + tickSetup := make(chan any) + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + Expect(d).To(Equal(time.Second)) + close(tickSetup) + return originalTick(d) + } + + lowPriority := -100 + highPriority := 0 + q.AddWithOpts(AddOpts{After: 0, Priority: &lowPriority}, "foo") + q.AddWithOpts(AddOpts{After: time.Second, Priority: &highPriority}, "prio") + + Eventually(tickSetup).Should(BeClosed()) + + forwardQueueTimeBy(1 * time.Second) + key, prio, _ := q.GetWithPriority() + + Expect(key).To(Equal("prio")) + Expect(prio).To(Equal(0)) + Expect(metrics.depth["test"]).To(Equal(map[int]int{-100: 1, 0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + Expect(metrics.retries["test"]).To(Equal(1)) + }) + + It("returns an item to a waiter as soon as it has one", func() { + q, metrics := newQueue() + defer q.ShutDown() + + retrieved := make(chan struct{}) + go func() { + defer GinkgoRecover() + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + close(retrieved) + }() + + // We are waiting for the GetWithPriority() call to be blocked + // on retrieving an item. As golang doesn't provide a way to + // check if something is listening on a channel without + // sending them a message, I can't think of a way to do this + // without sleeping. + time.Sleep(time.Second) + q.AddWithOpts(AddOpts{}, "foo") + Eventually(retrieved).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("returns multiple items with after in correct order", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + // What a bunch of bs. Deferring in here causes + // ginkgo to deadlock, presumably because it + // never returns after the defer. Not deferring + // hides the actual assertion result and makes + // it complain that there should be a defer. + // Move the assertion into a goroutine just to + // get around that mess. + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + + // This is not deterministic and depends on which of + // Add() or Spin() gets the lock first. + Expect(d).To(Or(Equal(200*time.Millisecond), Equal(time.Second))) + }() + <-done + return originalTick(d) + } + + retrievedItem := make(chan struct{}) + retrievedSecondItem := make(chan struct{}) + + go func() { + defer GinkgoRecover() + first, _, _ := q.GetWithPriority() + Expect(first).To(Equal("bar")) + close(retrievedItem) + + second, _, _ := q.GetWithPriority() + Expect(second).To(Equal("foo")) + close(retrievedSecondItem) + }() + + q.AddWithOpts(AddOpts{After: time.Second}, "foo") + q.AddWithOpts(AddOpts{After: 200 * time.Millisecond}, "bar") + + Consistently(retrievedItem).ShouldNot(BeClosed()) + + forwardQueueTimeBy(time.Second) + Eventually(retrievedItem).Should(BeClosed()) + Eventually(retrievedSecondItem).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + }) + + It("doesn't include non-ready items in Len()", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: time.Minute}, "foo") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{After: time.Minute}, "bar") + q.AddWithOpts(AddOpts{}, "bal") + + Expect(q.Len()).To(Equal(2)) + Expect(metrics.depth).To(HaveLen(1)) + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2})) + }) + + // ref: https://github.com/kubernetes-sigs/controller-runtime/issues/3239 + It("Get from priority queue might get stuck when the priority queue is shut down", func() { + q, _ := newQueue() + + q.Add("baz") + // shut down + q.ShutDown() + q.AddWithOpts(AddOpts{After: time.Second}, "foo") + + item, priority, isShutDown := q.GetWithPriority() + Expect(item).To(Equal("")) + Expect(priority).To(Equal(0)) + Expect(isShutDown).To(BeTrue()) + + item1, priority1, isShutDown := q.GetWithPriority() + Expect(item1).To(Equal("")) + Expect(priority1).To(Equal(0)) + Expect(isShutDown).To(BeTrue()) + }) + + It("Get from priority queue should get unblocked when the priority queue is shut down", func() { + q, _ := newQueue() + + getUnblocked := make(chan struct{}) + + go func() { + defer GinkgoRecover() + defer close(getUnblocked) + + item, priority, isShutDown := q.GetWithPriority() + Expect(item).To(Equal("")) + Expect(priority).To(Equal(0)) + Expect(isShutDown).To(BeTrue()) + }() + + // Verify the go routine above is now waiting for an item. + Eventually(q.waiters.Load).Should(Equal(int64(1))) + Consistently(getUnblocked).ShouldNot(BeClosed()) + + // shut down + q.ShutDown() + + // Verify the shutdown unblocked the go routine. + Eventually(getUnblocked).Should(BeClosed()) + }) + + It("items are included in Len() and the queueDepth metric once they are ready", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "foo") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "bar") + q.AddWithOpts(AddOpts{}, "bal") + + Expect(q.Len()).To(Equal(2)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2})) + metrics.mu.Unlock() + time.Sleep(time.Second) + Expect(q.Len()).To(Equal(4)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 4})) + metrics.mu.Unlock() + + // Drain queue + for range 4 { + item, _ := q.Get() + q.Done(item) + } + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + metrics.mu.Unlock() + + // Validate that doing it again still works to notice bugs with removing + // it from the queues becameReady tracking. + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "foo") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "bar") + q.AddWithOpts(AddOpts{}, "bal") + + Expect(q.Len()).To(Equal(2)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2})) + metrics.mu.Unlock() + time.Sleep(time.Second) + Expect(q.Len()).To(Equal(4)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 4})) + metrics.mu.Unlock() + }) + + It("returns many items", func() { + // This test ensures the queue is able to drain a large queue without panic'ing. + // In a previous version of the code we were calling queue.Delete within q.Ascend + // which led to a panic in queue.Ascend > iterate: + // "panic: runtime error: index out of range [0] with length 0" + q, _ := newQueue() + defer q.ShutDown() + + for range 20 { + for i := range 1000 { + rn := rand.N(100) + if rn < 10 { + q.AddWithOpts(AddOpts{After: time.Duration(rn) * time.Millisecond}, fmt.Sprintf("foo%d", i)) + } else { + q.AddWithOpts(AddOpts{Priority: &rn}, fmt.Sprintf("foo%d", i)) + } + } + + wg := sync.WaitGroup{} + for range 100 { // The panic only occurred relatively frequently with a high number of go routines. + wg.Add(1) + go func() { + defer wg.Done() + for range 10 { + obj, _, _ := q.GetWithPriority() + q.Done(obj) + } + }() + } + + wg.Wait() + } + }) + + It("updates metrics correctly for an item that gets initially added with after and then without", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: time.Hour}, "foo") + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{})) + metrics.mu.Unlock() + + q.AddWithOpts(AddOpts{}, "foo") + + Expect(q.Len()).To(Equal(1)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + metrics.mu.Unlock() + + // Get the item to ensure the codepath in + // `spin` for the metrics is passed by so + // that this starts failing if it incorrectly + // calls `metrics.add` again. + item, _ := q.Get() + Expect(item).To(Equal("foo")) + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + metrics.mu.Unlock() + }) + + It("Updates metrics correctly for an item whose requeueAfter expired that gets added again without requeueAfter", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: 50 * time.Millisecond}, "foo") + time.Sleep(100 * time.Millisecond) + + Expect(q.Len()).To(Equal(1)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + metrics.mu.Unlock() + + q.AddWithOpts(AddOpts{}, "foo") + Expect(q.Len()).To(Equal(1)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + metrics.mu.Unlock() + + // Get the item to ensure the codepath in + // `spin` for the metrics is passed by so + // that this starts failing if it incorrectly + // calls `metrics.add` again. + item, _ := q.Get() + Expect(item).To(Equal("foo")) + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + metrics.mu.Unlock() + }) + + It("When adding items with rateLimit, previous items' rateLimit should not affect subsequent items", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + q.rateLimiter = workqueue.NewTypedItemExponentialFailureRateLimiter[string](5*time.Millisecond, 1000*time.Second) + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + + Expect(d).To(Or(Equal(5*time.Millisecond), Equal(635*time.Millisecond))) + }() + <-done + return originalTick(d) + } + + retrievedItem := make(chan struct{}) + retrievedSecondItem := make(chan struct{}) + + go func() { + defer GinkgoRecover() + first, _, _ := q.GetWithPriority() + Expect(first).To(Equal("foo")) + close(retrievedItem) + + second, _, _ := q.GetWithPriority() + Expect(second).To(Equal("bar")) + close(retrievedSecondItem) + }() + + // after 7 calls, the next When("bar") call will return 640ms. + for range 7 { + q.rateLimiter.When("bar") + } + q.AddWithOpts(AddOpts{RateLimited: true}, "foo", "bar") + + Consistently(retrievedItem).ShouldNot(BeClosed()) + forwardQueueTimeBy(5 * time.Millisecond) + Eventually(retrievedItem).Should(BeClosed()) + + Consistently(retrievedSecondItem).ShouldNot(BeClosed()) + forwardQueueTimeBy(635 * time.Millisecond) + Eventually(retrievedSecondItem).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + Expect(metrics.retries["test"]).To(Equal(2)) + }) +}) + +func BenchmarkAddGetDone(b *testing.B) { + q := New[int]("") + defer q.ShutDown() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < 1000; i++ { + q.Add(i) + } + for range 1000 { + item, _ := q.Get() + q.Done(item) + } + } +} + +func BenchmarkAddOnly(b *testing.B) { + q := New[int]("") + defer q.ShutDown() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < 1000; i++ { + q.Add(i) + } + } +} + +func BenchmarkAddLockContended(b *testing.B) { + q := New[int]("") + defer q.ShutDown() + go func() { + for range 1000 { + item, _ := q.Get() + q.Done(item) + } + }() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < 1000; i++ { + q.Add(i) + } + } +} + +// TestFuzzPrioriorityQueue validates a set of basic +// invariants that should always be true: +// +// - The queue is threadsafe when multiple producers and consumers +// are involved +// - There are no deadlocks +// - An item is never handed out again before it is returned +// - Items in the queue are de-duplicated +// - max(existing priority, new priority) is used +func TestFuzzPriorityQueue(t *testing.T) { + t.Parallel() + + seed := time.Now().UnixNano() + t.Logf("seed: %d", seed) + f := fuzz.NewWithSeed(seed) + fuzzLock := sync.Mutex{} + fuzz := func(in any) { + fuzzLock.Lock() + defer fuzzLock.Unlock() + + f.Fuzz(in) + } + + inQueue := map[string]int{} + inQueueLock := sync.Mutex{} + + handedOut := sets.Set[string]{} + handedOutLock := sync.Mutex{} + + wg := sync.WaitGroup{} + q, metrics := newQueue() + + for range 10 { + wg.Add(1) + go func() { + defer wg.Done() + + for range 1000 { + opts, item := AddOpts{}, "" + + fuzz(&opts) + fuzz(&item) + + if opts.After > 100*time.Millisecond { + opts.After = 10 * time.Millisecond + } + opts.RateLimited = false + + func() { + inQueueLock.Lock() + defer inQueueLock.Unlock() + + q.AddWithOpts(opts, item) + if existingPriority, exists := inQueue[item]; !exists || existingPriority < ptr.Deref(opts.Priority, 0) { + inQueue[item] = ptr.Deref(opts.Priority, 0) + } + }() + } + }() + } + + for range 100 { + wg.Add(1) + + go func() { + defer wg.Done() + + for { + item, cont := func() (string, bool) { + inQueueLock.Lock() + defer inQueueLock.Unlock() + + if len(inQueue) == 0 { + return "", false + } + + item, priority, _ := q.GetWithPriority() + if expected := inQueue[item]; expected != priority { + t.Errorf("got priority %d, expected %d", priority, expected) + } + delete(inQueue, item) + return item, true + }() + + if !cont { + return + } + + func() { + handedOutLock.Lock() + defer handedOutLock.Unlock() + + if handedOut.Has(item) { + t.Errorf("item %s got handed out more than once", item) + } + + metrics.mu.Lock() + for priority, depth := range metrics.depth["test"] { + if depth < 0 { + t.Errorf("negative depth of %d for priority %d:", depth, priority) + } + } + + metrics.mu.Unlock() + handedOut.Insert(item) + }() + + func() { + handedOutLock.Lock() + defer handedOutLock.Unlock() + + handedOut.Delete(item) + q.Done(item) + }() + } + }() + } + + wg.Wait() +} + +func newQueueWithTimeForwarder() (_ *priorityqueue[string], _ *fakeMetricsProvider, forwardQueueTime func(time.Duration)) { + q, m := newQueue() + + now := time.Now().Round(time.Second) + nowLock := sync.Mutex{} + tick := make(chan time.Time) + + q.now = func() time.Time { + nowLock.Lock() + defer nowLock.Unlock() + return now + } + q.tick = func(d time.Duration) <-chan time.Time { + return tick + } + + return q, m, func(d time.Duration) { + nowLock.Lock() + now = now.Add(d) + nowLock.Unlock() + tick <- now + } +} + +func newQueue() (*priorityqueue[string], *fakeMetricsProvider) { + metrics := newFakeMetricsProvider() + q := New("test", func(o *Opts[string]) { + o.MetricProvider = metrics + }) + q.(*priorityqueue[string]).queue = &btreeInteractionValidator{ + bTree: q.(*priorityqueue[string]).queue, + } + + // validate that tick always gets a positive value as it will just return + // nil otherwise, which results in blocking forever. + upstreamTick := q.(*priorityqueue[string]).tick + q.(*priorityqueue[string]).tick = func(d time.Duration) <-chan time.Time { + if d <= 0 { + panic(fmt.Sprintf("got non-positive tick: %v", d)) + } + return upstreamTick(d) + } + return q.(*priorityqueue[string]), metrics +} + +type btreeInteractionValidator struct { + bTree[*item[string]] +} + +func (b *btreeInteractionValidator) ReplaceOrInsert(item *item[string]) (*item[string], bool) { + // There is no codepath that updates an item + item, alreadyExist := b.bTree.ReplaceOrInsert(item) + if alreadyExist { + panic(fmt.Sprintf("ReplaceOrInsert: item %v already existed", item)) + } + return item, alreadyExist +} + +func (b *btreeInteractionValidator) Delete(item *item[string]) (*item[string], bool) { + // There is no codepath that deletes an item that doesn't exist + old, existed := b.bTree.Delete(item) + if !existed { + panic(fmt.Sprintf("Delete: item %v not found", item)) + } + return old, existed +} diff --git a/pkg/doc.go b/pkg/doc.go index 65e2b71169..64693b4829 100644 --- a/pkg/doc.go +++ b/pkg/doc.go @@ -18,21 +18,21 @@ limitations under the License. Package pkg provides libraries for building Controllers. Controllers implement Kubernetes APIs and are foundational to building Operators, Workload APIs, Configuration APIs, Autoscalers, and more. -Client +# Client Client provides a Read + Write client for reading and writing Kubernetes objects. -Cache +# Cache Cache provides a Read client for reading objects from a local cache. A cache may register handlers to respond to events that update the cache. -Manager +# Manager Manager is required for creating a Controller and provides the Controller shared dependencies such as clients, caches, schemes, etc. Controllers should be Started through the Manager by calling Manager.Start. -Controller +# Controller Controller implements a Kubernetes API by responding to events (object Create, Update, Delete) and ensuring that the state specified in the Spec of the object matches the state of the system. This is called a reconcile. @@ -49,7 +49,7 @@ system must be read for each reconcile. * Controllers require Watches to be configured to enqueue reconcile.Requests in response to events. -Webhook +# Webhook Admission Webhooks are a mechanism for extending kubernetes APIs. Webhooks can be configured with target event type (object Create, Update, Delete), the API server will send AdmissionRequests to them @@ -62,7 +62,7 @@ Validating webhook is used to validate if an object meets certain requirements. * Admission Webhooks require Handler(s) to be provided to process the received AdmissionReview requests. -Reconciler +# Reconciler Reconciler is a function provided to a Controller that may be called at anytime with the Name and Namespace of an object. When called, the Reconciler will ensure that the state of the system matches what is specified in the object at the @@ -84,7 +84,7 @@ a mapping (e.g. owner references) that maps the object that triggers the reconci - e.g. it doesn't matter whether a ReplicaSet was created or updated, Reconciler will always compare the number of Pods in the system against what is specified in the object at the time it is called. -Source +# Source resource.Source is an argument to Controller.Watch that provides a stream of events. Events typically come from watching Kubernetes APIs (e.g. Pod Create, Update, Delete). @@ -97,7 +97,7 @@ through the Watch API. * Users SHOULD only use the provided Source implementations instead of implementing their own for nearly all cases. -EventHandler +# EventHandler handler.EventHandler is an argument to Controller.Watch that enqueues reconcile.Requests in response to events. @@ -117,7 +117,7 @@ type - e.g. map a Node event to objects that respond to cluster resize events. * Users SHOULD only use the provided EventHandler implementations instead of implementing their own for almost all cases. -Predicate +# Predicate predicate.Predicate is an optional argument to Controller.Watch that filters events. This allows common filters to be reused and composed. @@ -129,7 +129,7 @@ reused and composed. * Users SHOULD use the provided Predicate implementations, but MAY implement additional Predicates e.g. generation changed, label selectors changed etc. -PodController Diagram +# PodController Diagram Source provides event: @@ -137,20 +137,20 @@ Source provides event: EventHandler enqueues Request: -* &handler.EnqueueRequestForObject{} -> (reconcile.Request{types.NamespaceName{Name: "foo", Namespace: "bar"}}) +* &handler.EnqueueRequestForObject{} -> (reconcile.Request{types.NamespaceName{Namespace: "foo", Name: "bar"}}) Reconciler is called with the Request: -* Reconciler(reconcile.Request{types.NamespaceName{Name: "foo", Namespace: "bar"}}) +* Reconciler(reconcile.Request{types.NamespaceName{Namespace: "foo", Name: "bar"}}) -Usage +# Usage The following example shows creating a new Controller program which Reconciles ReplicaSet objects in response to Pod or ReplicaSet events. The Reconciler function simply adds a label to the ReplicaSet. See the examples/builtins/main.go for a usage example. -Controller Example +Controller Example: 1. Watch ReplicaSet and Pods Sources @@ -167,7 +167,7 @@ Owning ReplicaSet Namespace and Name. 2.3 Reconciler triggered by deletion of Pods from some other actor -> Read ReplicaSet and Pods, create replacement Pods. -Watching and EventHandling +# Watching and EventHandling Controllers may Watch multiple Kinds of objects (e.g. Pods, ReplicaSets and Deployments), but they reconcile only a single Type. When one Type of object must be updated in response to changes in another Type of object, @@ -185,7 +185,7 @@ Note: reconcile.Requests are deduplicated when they are enqueued. Many Pod Even may trigger only 1 reconcile invocation as each Event results in the Handler trying to enqueue the same reconcile.Request for the ReplicaSet. -Controller Writing Tips +# Controller Writing Tips Reconciler Runtime Complexity: diff --git a/pkg/envtest/binaries.go b/pkg/envtest/binaries.go new file mode 100644 index 0000000000..5110d32658 --- /dev/null +++ b/pkg/envtest/binaries.go @@ -0,0 +1,387 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "k8s.io/apimachinery/pkg/util/version" + "sigs.k8s.io/yaml" +) + +// DefaultBinaryAssetsIndexURL is the default index used in HTTPClient. +var DefaultBinaryAssetsIndexURL = "https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml" + +// SetupEnvtestDefaultBinaryAssetsDirectory returns the default location that setup-envtest uses to store envtest binaries. +// Setting BinaryAssetsDirectory to this directory allows sharing envtest binaries with setup-envtest. +// +// The directory is dependent on operating system: +// +// - Windows: %LocalAppData%\kubebuilder-envtest +// - OSX: ~/Library/Application Support/io.kubebuilder.envtest +// - Others: ${XDG_DATA_HOME:-~/.local/share}/kubebuilder-envtest +// +// Otherwise, it errors out. Note that these paths must not be relied upon +// manually. +func SetupEnvtestDefaultBinaryAssetsDirectory() (string, error) { + var baseDir string + + // find the base data directory + switch runtime.GOOS { + case "windows": + baseDir = os.Getenv("LocalAppData") + if baseDir == "" { + return "", errors.New("%LocalAppData% is not defined") + } + case "darwin": + homeDir := os.Getenv("HOME") + if homeDir == "" { + return "", errors.New("$HOME is not defined") + } + baseDir = filepath.Join(homeDir, "Library/Application Support") + default: + baseDir = os.Getenv("XDG_DATA_HOME") + if baseDir == "" { + homeDir := os.Getenv("HOME") + if homeDir == "" { + return "", errors.New("neither $XDG_DATA_HOME nor $HOME are defined") + } + baseDir = filepath.Join(homeDir, ".local/share") + } + } + + // append our program-specific dir to it (OSX has a slightly different + // convention so try to follow that). + switch runtime.GOOS { + case "darwin", "ios": + return filepath.Join(baseDir, "io.kubebuilder.envtest", "k8s"), nil + default: + return filepath.Join(baseDir, "kubebuilder-envtest", "k8s"), nil + } +} + +// index represents an index of envtest binary archives. Example: +// +// releases: +// v1.28.0: +// envtest-v1.28.0-darwin-amd64.tar.gz: +// hash: +// selfLink: +type index struct { + // Releases maps Kubernetes versions to Releases (envtest archives). + Releases map[string]release `json:"releases"` +} + +// release maps an archive name to an archive. +type release map[string]archive + +// archive contains the self link to an archive and its hash. +type archive struct { + Hash string `json:"hash"` + SelfLink string `json:"selfLink"` +} + +// parseKubernetesVersion returns: +// 1. the SemVer form of s when it refers to a specific Kubernetes release, or +// 2. the major and minor portions of s when it refers to a release series, or +// 3. an error +func parseKubernetesVersion(s string) (exact string, major, minor uint, err error) { + if v, err := version.ParseSemantic(s); err == nil { + return v.String(), 0, 0, nil + } + + // See two parseable components and nothing else. + if v, err := version.ParseGeneric(s); err == nil && len(v.Components()) == 2 { + if v.String() == strings.TrimPrefix(s, "v") { + return "", v.Major(), v.Minor(), nil + } + } + + return "", 0, 0, fmt.Errorf("could not parse %q as version", s) +} + +func downloadBinaryAssets(ctx context.Context, binaryAssetsDirectory, binaryAssetsVersion, binaryAssetsIndexURL string) (string, string, string, error) { + if binaryAssetsIndexURL == "" { + binaryAssetsIndexURL = DefaultBinaryAssetsIndexURL + } + + downloadRootDir := binaryAssetsDirectory + if downloadRootDir == "" { + var err error + if downloadRootDir, err = os.MkdirTemp("", "envtest-binaries-"); err != nil { + return "", "", "", fmt.Errorf("failed to create tmp directory for envtest binaries: %w", err) + } + } + + var binaryAssetsIndex *index + switch exact, major, minor, err := parseKubernetesVersion(binaryAssetsVersion); { + case binaryAssetsVersion != "" && err != nil: + return "", "", "", err + + case binaryAssetsVersion != "" && exact != "": + // Look for these specific binaries locally before downloading them from the release index. + // Use the canonical form of the version from here on. + binaryAssetsVersion = "v" + exact + + case binaryAssetsVersion == "" || major != 0 || minor != 0: + // Select a stable version from the release index before continuing. + binaryAssetsIndex, err = getIndex(ctx, binaryAssetsIndexURL) + if err != nil { + return "", "", "", err + } + + binaryAssetsVersion, err = latestStableVersionFromIndex(binaryAssetsIndex, major, minor) + if err != nil { + return "", "", "", err + } + } + + // Storing the envtest binaries in a directory structure that is compatible with setup-envtest. + // This makes it possible to share the envtest binaries with setup-envtest if the BinaryAssetsDirectory is set to SetupEnvtestDefaultBinaryAssetsDirectory(). + downloadDir := path.Join(downloadRootDir, fmt.Sprintf("%s-%s-%s", strings.TrimPrefix(binaryAssetsVersion, "v"), runtime.GOOS, runtime.GOARCH)) + if !fileExists(downloadDir) { + if err := os.MkdirAll(downloadDir, 0700); err != nil { + return "", "", "", fmt.Errorf("failed to create directory %q for envtest binaries: %w", downloadDir, err) + } + } + + apiServerPath := path.Join(downloadDir, "kube-apiserver") + etcdPath := path.Join(downloadDir, "etcd") + kubectlPath := path.Join(downloadDir, "kubectl") + + if fileExists(apiServerPath) && fileExists(etcdPath) && fileExists(kubectlPath) { + // Nothing to do if the binaries already exist. + return apiServerPath, etcdPath, kubectlPath, nil + } + + // Get Index if we didn't have to get it above to get the latest stable version. + if binaryAssetsIndex == nil { + var err error + binaryAssetsIndex, err = getIndex(ctx, binaryAssetsIndexURL) + if err != nil { + return "", "", "", err + } + } + + buf := &bytes.Buffer{} + if err := downloadBinaryAssetsArchive(ctx, binaryAssetsIndex, binaryAssetsVersion, buf); err != nil { + return "", "", "", err + } + + gzStream, err := gzip.NewReader(buf) + if err != nil { + return "", "", "", fmt.Errorf("failed to create gzip reader to extract envtest binaries: %w", err) + } + tarReader := tar.NewReader(gzStream) + + var header *tar.Header + for header, err = tarReader.Next(); err == nil; header, err = tarReader.Next() { + if header.Typeflag != tar.TypeReg { + // Skip non-regular file entry in archive. + continue + } + + // Just dump all files directly into the download directory, ignoring the prefixed directory paths. + // We also ignore bits for the most part (except for X). + fileName := filepath.Base(header.Name) + perms := 0555 & header.Mode // make sure we're at most r+x + + // Setting O_EXCL to get an error if the file already exists. + f, err := os.OpenFile(path.Join(downloadDir, fileName), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_TRUNC, os.FileMode(perms)) + if err != nil { + if os.IsExist(err) { + // Nothing to do if the file already exists. We assume another process created the file concurrently. + continue + } + return "", "", "", fmt.Errorf("failed to create file %s in directory %s: %w", fileName, downloadDir, err) + } + if err := func() error { + defer f.Close() + if _, err := io.Copy(f, tarReader); err != nil { + return fmt.Errorf("failed to write file %s in directory %s: %w", fileName, downloadDir, err) + } + return nil + }(); err != nil { + return "", "", "", fmt.Errorf("failed to close file %s in directory %s: %w", fileName, downloadDir, err) + } + } + + return apiServerPath, etcdPath, kubectlPath, nil +} + +func fileExists(path string) bool { + if _, err := os.Stat(path); err == nil { + return true + } + return false +} + +func downloadBinaryAssetsArchive(ctx context.Context, index *index, version string, out io.Writer) error { + archives, ok := index.Releases[version] + if !ok { + return fmt.Errorf("failed to find envtest binaries for version %s", version) + } + + archiveName := fmt.Sprintf("envtest-%s-%s-%s.tar.gz", version, runtime.GOOS, runtime.GOARCH) + archive, ok := archives[archiveName] + if !ok { + return fmt.Errorf("failed to find envtest binaries for version %s with archiveName %s", version, archiveName) + } + + archiveURL, err := url.Parse(archive.SelfLink) + if err != nil { + return fmt.Errorf("failed to parse envtest binaries archive URL %q: %w", archiveURL, err) + } + + req, err := http.NewRequestWithContext(ctx, "GET", archiveURL.String(), nil) + if err != nil { + return fmt.Errorf("failed to create request to download %s: %w", archiveURL.String(), err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download %s: %w", archiveURL.String(), err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("failed to download %s, got status %q", archiveURL.String(), resp.Status) + } + + return readBody(resp, out, archiveName, archive.Hash) +} + +// latestStableVersionFromIndex returns the version with highest [precedence] in index that is not a prerelease. +// When either major or minor are not zero, the returned version will have those major and minor versions. +// Note that the version cannot be limited to 0.0.x this way. +// +// It is an error when there is no appropriate version in index. +// +// [precedence]: https://semver.org/spec/v2.0.0.html#spec-item-11 +func latestStableVersionFromIndex(index *index, major, minor uint) (string, error) { + if len(index.Releases) == 0 { + return "", fmt.Errorf("failed to find latest stable version from index: index is empty") + } + + var found *version.Version + for releaseVersion := range index.Releases { + v, err := version.ParseSemantic(releaseVersion) + if err != nil { + return "", fmt.Errorf("failed to parse version %q: %w", releaseVersion, err) + } + + // Filter out pre-releases. + if len(v.PreRelease()) > 0 { + continue + } + + // Filter on release series, if any. + if (major != 0 || minor != 0) && (v.Major() != major || v.Minor() != minor) { + continue + } + + if found == nil || v.GreaterThan(found) { + found = v + } + } + + if found == nil { + search := "any" + if major != 0 || minor != 0 { + search = fmt.Sprint(major, ".", minor) + } + + return "", fmt.Errorf("failed to find latest stable version from index: index does not have %s stable versions", search) + } + + return "v" + found.String(), nil +} + +func getIndex(ctx context.Context, indexURL string) (*index, error) { + loc, err := url.Parse(indexURL) + if err != nil { + return nil, fmt.Errorf("unable to parse index URL: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return nil, fmt.Errorf("unable to construct request to get index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("unable to perform request to get index: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unable to get index -- got status %q", resp.Status) + } + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to get index -- unable to read body %w", err) + } + + var index index + if err := yaml.Unmarshal(responseBody, &index); err != nil { + return nil, fmt.Errorf("unable to unmarshal index: %w", err) + } + return &index, nil +} + +func readBody(resp *http.Response, out io.Writer, archiveName string, expectedHash string) error { + // Stream in chunks to do the checksum + buf := make([]byte, 32*1024) // 32KiB, same as io.Copy + hasher := sha512.New() + + for cont := true; cont; { + amt, err := resp.Body.Read(buf) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("unable read next chunk of %s: %w", archiveName, err) + } + if amt > 0 { + // checksum never returns errors according to docs + hasher.Write(buf[:amt]) + if _, err := out.Write(buf[:amt]); err != nil { + return fmt.Errorf("unable write next chunk of %s: %w", archiveName, err) + } + } + cont = amt > 0 && !errors.Is(err, io.EOF) + } + + actualHash := hex.EncodeToString(hasher.Sum(nil)) + if actualHash != expectedHash { + return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (expected)", archiveName, actualHash, expectedHash) + } + + return nil +} diff --git a/pkg/envtest/binaries_test.go b/pkg/envtest/binaries_test.go new file mode 100644 index 0000000000..aa83963381 --- /dev/null +++ b/pkg/envtest/binaries_test.go @@ -0,0 +1,357 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/rand" + "crypto/sha512" + "encoding/hex" + "fmt" + "net/http" + "os" + "path" + "runtime" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/ghttp" + "sigs.k8s.io/yaml" +) + +func TestParseKubernetesVersion(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + inputs []string + + expectError string + expectExact bool + expectSeriesMajor uint + expectSeriesMinor uint + }{ + { + name: `SemVer and "v" prefix are exact`, + inputs: []string{ + "1.2.3", "v1.2.3", "v1.30.2", "v1.31.0-beta.0", "v1.33.0-alpha.2", + }, + expectExact: true, + }, + { + name: "empty string is not a version", + inputs: []string{""}, + expectError: "could not parse", + }, + { + name: "leading zeroes are not a version", + inputs: []string{ + "01.2.0", "00001.2.3", "1.2.03", "v01.02.0003", + }, + expectError: "could not parse", + }, + { + name: "weird stuff is not a version", + inputs: []string{ + "asdf", "version", "vegeta4", "the.1", "2ne1", "=7.8.9", "10.x", "*", + "0.0001", "1.00002", "v1.2anything", "1.2.x", "1.2.z", "1.2.*", + }, + expectError: "could not parse", + }, + { + name: "one number is not a version", + inputs: []string{ + "1", "v1", "v001", "1.", "v1.", "1.x", + }, + expectError: "could not parse", + }, + { + name: "two numbers are a release series", + inputs: []string{"0.1", "v0.1"}, + + expectSeriesMajor: 0, + expectSeriesMinor: 1, + }, + { + name: "two numbers are a release series", + inputs: []string{"1.2", "v1.2"}, + + expectSeriesMajor: 1, + expectSeriesMinor: 2, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for _, input := range tc.inputs { + exact, major, minor, err := parseKubernetesVersion(input) + + if tc.expectError != "" && err == nil { + t.Errorf("expected error %q, got none", tc.expectError) + } + if tc.expectError != "" && !strings.Contains(err.Error(), tc.expectError) { + t.Errorf("expected error %q, got %q", tc.expectError, err) + } + if tc.expectError == "" && err != nil { + t.Errorf("expected no error, got %q", err) + continue + } + + if tc.expectExact { + if expected := strings.TrimPrefix(input, "v"); exact != expected { + t.Errorf("expected canonical %q for %q, got %q", expected, input, exact) + } + if major != 0 || minor != 0 { + t.Errorf("expected no release series for %q, got (%v, %v)", input, major, minor) + } + continue + } + + if major != tc.expectSeriesMajor { + t.Errorf("expected major %v for %q, got %v", tc.expectSeriesMajor, input, major) + } + if minor != tc.expectSeriesMinor { + t.Errorf("expected minor %v for %q, got %v", tc.expectSeriesMinor, input, minor) + } + if exact != "" { + t.Errorf("expected no canonical version for %q, got %q", input, exact) + } + } + }) + } +} + +var _ = Describe("Test download binaries", func() { + var downloadDirectory string + var server *ghttp.Server + + BeforeEach(func() { + downloadDirectory = GinkgoT().TempDir() + + server = ghttp.NewServer() + DeferCleanup(func() { + server.Close() + }) + setupServer(server) + }) + + It("should download binaries of latest stable version", func(ctx SpecContext) { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(ctx, downloadDirectory, "", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Verify latest stable version (v1.32.0) was downloaded + versionDownloadDirectory := path.Join(downloadDirectory, fmt.Sprintf("1.32.0-%s-%s", runtime.GOOS, runtime.GOARCH)) + Expect(apiServerPath).To(Equal(path.Join(versionDownloadDirectory, "kube-apiserver"))) + Expect(etcdPath).To(Equal(path.Join(versionDownloadDirectory, "etcd"))) + Expect(kubectlPath).To(Equal(path.Join(versionDownloadDirectory, "kubectl"))) + + dirEntries, err := os.ReadDir(versionDownloadDirectory) + Expect(err).ToNot(HaveOccurred()) + var actualFiles []string + for _, e := range dirEntries { + actualFiles = append(actualFiles, e.Name()) + } + Expect(actualFiles).To(ConsistOf("some-file")) + }) + + It("should download binaries of an exact version", func(ctx SpecContext) { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(ctx, downloadDirectory, "v1.31.0", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Verify exact version (v1.31.0) was downloaded + versionDownloadDirectory := path.Join(downloadDirectory, fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)) + Expect(apiServerPath).To(Equal(path.Join(versionDownloadDirectory, "kube-apiserver"))) + Expect(etcdPath).To(Equal(path.Join(versionDownloadDirectory, "etcd"))) + Expect(kubectlPath).To(Equal(path.Join(versionDownloadDirectory, "kubectl"))) + + dirEntries, err := os.ReadDir(versionDownloadDirectory) + Expect(err).ToNot(HaveOccurred()) + var actualFiles []string + for _, e := range dirEntries { + actualFiles = append(actualFiles, e.Name()) + } + Expect(actualFiles).To(ConsistOf("some-file")) + }) + + It("should download binaries of latest stable version of a release series", func(ctx SpecContext) { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(ctx, downloadDirectory, "1.31", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Verify stable version (v1.31.4) was downloaded + versionDownloadDirectory := path.Join(downloadDirectory, fmt.Sprintf("1.31.4-%s-%s", runtime.GOOS, runtime.GOARCH)) + Expect(apiServerPath).To(Equal(path.Join(versionDownloadDirectory, "kube-apiserver"))) + Expect(etcdPath).To(Equal(path.Join(versionDownloadDirectory, "etcd"))) + Expect(kubectlPath).To(Equal(path.Join(versionDownloadDirectory, "kubectl"))) + + dirEntries, err := os.ReadDir(versionDownloadDirectory) + Expect(err).ToNot(HaveOccurred()) + var actualFiles []string + for _, e := range dirEntries { + actualFiles = append(actualFiles, e.Name()) + } + Expect(actualFiles).To(ConsistOf("some-file")) + }) + + It("should error when the asset version is not a version", func(ctx SpecContext) { + _, _, _, err := downloadBinaryAssets(ctx, downloadDirectory, "wonky", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).To(MatchError(`could not parse "wonky" as version`)) + }) + + It("should error when the asset version is not in the index", func(ctx SpecContext) { + _, _, _, err := downloadBinaryAssets(ctx, downloadDirectory, "v1.5.0", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).To(MatchError("failed to find envtest binaries for version v1.5.0")) + + _, _, _, err = downloadBinaryAssets(ctx, downloadDirectory, "v1.5", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).To(MatchError("failed to find latest stable version from index: index does not have 1.5 stable versions")) + }) +}) + +var ( + envtestBinaryArchives = index{ + Releases: map[string]release{ + "v1.32.0": map[string]archive{ + "envtest-v1.32.0-darwin-amd64.tar.gz": {}, + "envtest-v1.32.0-darwin-arm64.tar.gz": {}, + "envtest-v1.32.0-linux-amd64.tar.gz": {}, + "envtest-v1.32.0-linux-arm64.tar.gz": {}, + "envtest-v1.32.0-linux-ppc64le.tar.gz": {}, + "envtest-v1.32.0-linux-s390x.tar.gz": {}, + "envtest-v1.32.0-windows-amd64.tar.gz": {}, + }, + "v1.31.4": map[string]archive{ + "envtest-v1.31.4-darwin-amd64.tar.gz": {}, + "envtest-v1.31.4-darwin-arm64.tar.gz": {}, + "envtest-v1.31.4-linux-amd64.tar.gz": {}, + "envtest-v1.31.4-linux-arm64.tar.gz": {}, + "envtest-v1.31.4-linux-ppc64le.tar.gz": {}, + "envtest-v1.31.4-linux-s390x.tar.gz": {}, + "envtest-v1.31.4-windows-amd64.tar.gz": {}, + }, + "v1.31.0": map[string]archive{ + "envtest-v1.31.0-darwin-amd64.tar.gz": {}, + "envtest-v1.31.0-darwin-arm64.tar.gz": {}, + "envtest-v1.31.0-linux-amd64.tar.gz": {}, + "envtest-v1.31.0-linux-arm64.tar.gz": {}, + "envtest-v1.31.0-linux-ppc64le.tar.gz": {}, + "envtest-v1.31.0-linux-s390x.tar.gz": {}, + "envtest-v1.31.0-windows-amd64.tar.gz": {}, + }, + }, + } +) + +func setupServer(server *ghttp.Server) { + itemsHTTP := makeArchives(envtestBinaryArchives) + + // The index from itemsHTTP contains only relative SelfLinks. + // finalIndex will contain the full links based on server.Addr(). + finalIndex := index{ + Releases: map[string]release{}, + } + + for releaseVersion, releases := range itemsHTTP.index.Releases { + finalIndex.Releases[releaseVersion] = release{} + + for archiveName, a := range releases { + finalIndex.Releases[releaseVersion][archiveName] = archive{ + Hash: a.Hash, + SelfLink: fmt.Sprintf("http://%s/%s", server.Addr(), a.SelfLink), + } + content := itemsHTTP.contents[archiveName] + + // Note: Using the relative path from archive here instead of the full path. + server.RouteToHandler("GET", "/"+a.SelfLink, func(resp http.ResponseWriter, req *http.Request) { + resp.WriteHeader(http.StatusOK) + Expect(resp.Write(content)).To(Equal(len(content))) + }) + } + } + + indexYAML, err := yaml.Marshal(finalIndex) + Expect(err).ToNot(HaveOccurred()) + + server.RouteToHandler("GET", "/envtest-releases.yaml", ghttp.RespondWith( + http.StatusOK, + indexYAML, + )) +} + +type itemsHTTP struct { + index index + contents map[string][]byte +} + +func makeArchives(i index) itemsHTTP { + // This creates a new copy of the index so modifying the index + // in some tests doesn't affect others. + res := itemsHTTP{ + index: index{ + Releases: map[string]release{}, + }, + contents: map[string][]byte{}, + } + + for releaseVersion, releases := range i.Releases { + res.index.Releases[releaseVersion] = release{} + for archiveName := range releases { + var chunk [1024 * 48]byte // 1.5 times our chunk read size in GetVersion + copy(chunk[:], archiveName) + if _, err := rand.Read(chunk[len(archiveName):]); err != nil { + panic(err) + } + content, hash := makeArchive(chunk[:]) + + res.index.Releases[releaseVersion][archiveName] = archive{ + Hash: hash, + // Note: Only storing the name of the archive for now. + // This will be expanded later to a full URL once the server is running. + SelfLink: archiveName, + } + res.contents[archiveName] = content + } + } + return res +} + +func makeArchive(contents []byte) ([]byte, string) { + out := new(bytes.Buffer) + gzipWriter := gzip.NewWriter(out) + tarWriter := tar.NewWriter(gzipWriter) + err := tarWriter.WriteHeader(&tar.Header{ + Name: "controller-tools/envtest/some-file", + Size: int64(len(contents)), + Mode: 0777, // so we can check that we fix this later + }) + if err != nil { + panic(err) + } + _, err = tarWriter.Write(contents) + if err != nil { + panic(err) + } + tarWriter.Close() + gzipWriter.Close() + content := out.Bytes() + // controller-tools is using sha512 + hash := sha512.Sum512(content) + hashEncoded := hex.EncodeToString(hash[:]) + return content, hashEncoded +} diff --git a/pkg/envtest/crd.go b/pkg/envtest/crd.go index a15c1dacaa..8ed2224cfe 100644 --- a/pkg/envtest/crd.go +++ b/pkg/envtest/crd.go @@ -20,9 +20,9 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "time" @@ -38,7 +38,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/yaml" "sigs.k8s.io/controller-runtime/pkg/client" @@ -84,15 +84,17 @@ type CRDInstallOptions struct { WebhookOptions WebhookInstallOptions } -const defaultPollInterval = 100 * time.Millisecond -const defaultMaxWait = 10 * time.Second +const ( + defaultPollInterval = 100 * time.Millisecond + defaultMaxWait = 10 * time.Second +) // InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory. func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDefinition, error) { defaultCRDOptions(&options) // Read the CRD yamls into options.CRDs - if err := readCRDFiles(&options); err != nil { + if err := ReadCRDFiles(&options); err != nil { return nil, fmt.Errorf("unable to read CRD files: %w", err) } @@ -113,8 +115,8 @@ func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensio return options.CRDs, nil } -// readCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs. -func readCRDFiles(options *CRDInstallOptions) error { +// ReadCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs. +func ReadCRDFiles(options *CRDInstallOptions) error { if len(options.Paths) > 0 { crdList, err := renderCRDs(options) if err != nil { @@ -142,7 +144,7 @@ func defaultCRDOptions(o *CRDInstallOptions) { // WaitForCRDs waits for the CRDs to appear in discovery. func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefinition, options CRDInstallOptions) error { // Add each CRD to a map of GroupVersion to Resource - waitingFor := map[schema.GroupVersion]*sets.String{} + waitingFor := map[schema.GroupVersion]*sets.Set[string]{} for _, crd := range crds { gvs := []schema.GroupVersion{} for _, version := range crd.Spec.Versions { @@ -155,7 +157,7 @@ func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefi log.V(1).Info("adding API in waitlist", "GV", gv) if _, found := waitingFor[gv]; !found { // Initialize the set - waitingFor[gv] = &sets.String{} + waitingFor[gv] = &sets.Set[string]{} } // Add the Resource waitingFor[gv].Insert(crd.Spec.Names.Plural) @@ -164,7 +166,7 @@ func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefi // Poll until all resources are found in discovery p := &poller{config: config, waitingFor: waitingFor} - return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll) + return wait.PollUntilContextTimeout(context.TODO(), options.PollInterval, options.MaxTime, true, p.poll) } // poller checks if all the resources have been found in discovery, and returns false if not. @@ -173,11 +175,11 @@ type poller struct { config *rest.Config // waitingFor is the map of resources keyed by group version that have not yet been found in discovery - waitingFor map[schema.GroupVersion]*sets.String + waitingFor map[schema.GroupVersion]*sets.Set[string] } // poll checks if all the resources have been found in discovery, and returns false if not. -func (p *poller) poll() (done bool, err error) { +func (p *poller) poll(ctx context.Context) (done bool, err error) { // Create a new clientset to avoid any client caching of discovery cs, err := clientset.NewForConfig(p.config) if err != nil { @@ -215,7 +217,7 @@ func (p *poller) poll() (done bool, err error) { // UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory. func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error { // Read the CRD yamls into options.CRDs - if err := readCRDFiles(&options); err != nil { + if err := ReadCRDFiles(&options); err != nil { return err } @@ -227,7 +229,6 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error { // Uninstall each CRD for _, crd := range options.CRDs { - crd := crd log.V(1).Info("uninstalling CRD", "crd", crd.GetName()) if err := cs.Delete(context.TODO(), crd); err != nil { // If CRD is not found, we can consider success @@ -249,7 +250,6 @@ func CreateCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefin // Create each CRD for _, crd := range crds { - crd := crd log.V(1).Info("installing CRD", "crd", crd.GetName()) existingCrd := crd.DeepCopy() err := cs.Get(context.TODO(), client.ObjectKey{Name: crd.GetName()}, existingCrd) @@ -278,12 +278,6 @@ func CreateCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefin // renderCRDs iterate through options.Paths and extract all CRD files. func renderCRDs(options *CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDefinition, error) { - var ( - err error - info os.FileInfo - files []os.FileInfo - ) - type GVKN struct { GVK schema.GroupVersionKind Name string @@ -292,7 +286,12 @@ func renderCRDs(options *CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDe crds := map[GVKN]*apiextensionsv1.CustomResourceDefinition{} for _, path := range options.Paths { - var filePath = path + var ( + err error + info os.FileInfo + files []string + filePath = path + ) // Return the error if ErrorIfPathMissing exists if info, err = os.Stat(path); os.IsNotExist(err) { @@ -303,9 +302,15 @@ func renderCRDs(options *CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDe } if !info.IsDir() { - filePath, files = filepath.Dir(path), []os.FileInfo{info} - } else if files, err = ioutil.ReadDir(path); err != nil { - return nil, err + filePath, files = filepath.Dir(path), []string{info.Name()} + } else { + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + for _, e := range entries { + files = append(files, e.Name()) + } } log.V(1).Info("reading CRDs from path", "path", path) @@ -357,19 +362,26 @@ func modifyConversionWebhooks(crds []*apiextensionsv1.CustomResourceDefinition, if err != nil { return err } - url := pointer.StringPtr(fmt.Sprintf("https://%s/convert", hostPort)) + url := ptr.To(fmt.Sprintf("https://%s/convert", hostPort)) for i := range crds { // Continue if we're preserving unknown fields. if crds[i].Spec.PreserveUnknownFields { continue } - // Continue if the GroupKind isn't registered as being convertible. - if _, ok := convertibles[schema.GroupKind{ - Group: crds[i].Spec.Group, - Kind: crds[i].Spec.Names.Kind, - }]; !ok { - continue + if !webhookOptions.IgnoreSchemeConvertible { + // Continue if the GroupKind isn't registered as being convertible, + // and remove any existing conversion webhooks if they exist. + // This is to prevent the CRD from being rejected by the apiserver, usually + // manifests that are generated by controller-gen will have a conversion + // webhook set, but we don't want to enable it if the type isn't registered. + if _, ok := convertibles[schema.GroupKind{ + Group: crds[i].Spec.Group, + Kind: crds[i].Spec.Names.Kind, + }]; !ok { + crds[i].Spec.Conversion = nil + continue + } } if crds[i].Spec.Conversion == nil { crds[i].Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ @@ -389,7 +401,7 @@ func modifyConversionWebhooks(crds []*apiextensionsv1.CustomResourceDefinition, } // readCRDs reads the CRDs from files and Unmarshals them into structs. -func readCRDs(basePath string, files []os.FileInfo) ([]*apiextensionsv1.CustomResourceDefinition, error) { +func readCRDs(basePath string, files []string) ([]*apiextensionsv1.CustomResourceDefinition, error) { var crds []*apiextensionsv1.CustomResourceDefinition // White list the file extensions that may contain CRDs @@ -397,12 +409,12 @@ func readCRDs(basePath string, files []os.FileInfo) ([]*apiextensionsv1.CustomRe for _, file := range files { // Only parse allowlisted file types - if !crdExts.Has(filepath.Ext(file.Name())) { + if !crdExts.Has(filepath.Ext(file)) { continue } // Unmarshal CRDs from file into structs - docs, err := readDocuments(filepath.Join(basePath, file.Name())) + docs, err := readDocuments(filepath.Join(basePath, file)) if err != nil { return nil, err } @@ -419,14 +431,14 @@ func readCRDs(basePath string, files []os.FileInfo) ([]*apiextensionsv1.CustomRe crds = append(crds, crd) } - log.V(1).Info("read CRDs from file", "file", file.Name()) + log.V(1).Info("read CRDs from file", "file", file) } return crds, nil } // readDocuments reads documents from file. func readDocuments(fp string) ([][]byte, error) { - b, err := ioutil.ReadFile(fp) //nolint:gosec + b, err := os.ReadFile(fp) if err != nil { return nil, err } @@ -437,7 +449,7 @@ func readDocuments(fp string) ([][]byte, error) { // Read document doc, err := reader.Read() if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { break } diff --git a/pkg/envtest/crd_test.go b/pkg/envtest/crd_test.go new file mode 100644 index 0000000000..a1406615d6 --- /dev/null +++ b/pkg/envtest/crd_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/sets" +) + +var _ = Describe("Test", func() { + Describe("readCRDFiles", func() { + It("should not mix up files from different directories", func() { + opt := CRDInstallOptions{ + Paths: []string{ + "testdata/crds", + "testdata/crdv1_original", + }, + } + err := ReadCRDFiles(&opt) + Expect(err).NotTo(HaveOccurred()) + + expectedCRDs := sets.NewString( + "frigates.ship.example.com", + "configs.foo.example.com", + "drivers.crew.example.com", + ) + + foundCRDs := sets.NewString() + for _, crd := range opt.CRDs { + foundCRDs.Insert(crd.Name) + } + + Expect(expectedCRDs).To(Equal(foundCRDs)) + }) + }) +}) diff --git a/pkg/envtest/envtest_suite_test.go b/pkg/envtest/envtest_suite_test.go index 0d5bb0eae2..f7788bf090 100644 --- a/pkg/envtest/envtest_suite_test.go +++ b/pkg/envtest/envtest_suite_test.go @@ -19,20 +19,18 @@ package envtest import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Envtest Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Envtest Suite") } var env *Environment @@ -44,7 +42,7 @@ var _ = BeforeSuite(func() { initializeWebhookInEnvironment() _, err := env.Start() Expect(err).NotTo(HaveOccurred()) -}, StartTimeout) +}) func initializeWebhookInEnvironment() { namespacedScopeV1 := admissionv1.NamespacedScope @@ -133,4 +131,4 @@ func initializeWebhookInEnvironment() { var _ = AfterSuite(func() { Expect(env.Stop()).NotTo(HaveOccurred()) -}, StopTimeout) +}) diff --git a/pkg/envtest/envtest_test.go b/pkg/envtest/envtest_test.go index 11eedca0ac..ce3e9a4d3f 100644 --- a/pkg/envtest/envtest_test.go +++ b/pkg/envtest/envtest_test.go @@ -17,18 +17,17 @@ limitations under the License. package envtest import ( - "context" "path/filepath" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" - + "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -55,30 +54,29 @@ var _ = Describe("Test", func() { }) // Cleanup CRDs - AfterEach(func() { + AfterEach(func(ctx SpecContext) { for _, crd := range crds { - crd := crd // Delete only if CRD exists. crdObjectKey := client.ObjectKey{ Name: crd.GetName(), } var placeholder apiextensionsv1.CustomResourceDefinition - if err = c.Get(context.TODO(), crdObjectKey, &placeholder); err != nil && + if err = c.Get(ctx, crdObjectKey, &placeholder); err != nil && apierrors.IsNotFound(err) { // CRD doesn't need to be deleted. continue } Expect(err).NotTo(HaveOccurred()) - Expect(c.Delete(context.TODO(), crd)).To(Succeed()) + Expect(c.Delete(ctx, crd)).To(Succeed()) Eventually(func() bool { - err := c.Get(context.TODO(), crdObjectKey, &placeholder) + err := c.Get(ctx, crdObjectKey, &placeholder) return apierrors.IsNotFound(err) }, 5*time.Second).Should(BeTrue()) } }, teardownTimeoutSeconds) Describe("InstallCRDs", func() { - It("should install the unserved CRDs into the cluster", func() { + It("should install the unserved CRDs into the cluster", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata", "crds", "examplecrd_unserved.yaml")}, }) @@ -87,7 +85,7 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "frigates.ship.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "frigates.ship.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Frigate")) @@ -116,7 +114,7 @@ var _ = Describe("Test", func() { ) Expect(err).NotTo(HaveOccurred()) }) - It("should install the CRDs into the cluster using directory", func() { + It("should install the CRDs into the cluster using directory", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{validDirectory}, }) @@ -125,27 +123,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -243,16 +241,16 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) - }, 5) + }) - It("should install the CRDs into the cluster using file", func() { + It("should install the CRDs into the cluster using file", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata", "crds", "examplecrd3.yaml")}, }) Expect(err).NotTo(HaveOccurred()) crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "configs.foo.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "configs.foo.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Config")) @@ -278,7 +276,7 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) - }, 10) + }) It("should be able to install CRDs using multiple files", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ @@ -289,9 +287,9 @@ var _ = Describe("Test", func() { }) Expect(err).NotTo(HaveOccurred()) Expect(crds).To(HaveLen(2)) - }, 10) + }) - It("should filter out already existent CRD", func() { + It("should filter out already existent CRD", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{ filepath.Join(".", "testdata"), @@ -301,7 +299,7 @@ var _ = Describe("Test", func() { Expect(err).NotTo(HaveOccurred()) crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) @@ -329,26 +327,26 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) - }, 10) + }) It("should not return an not error if the directory doesn't exist", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{Paths: []string{invalidDirectory}}) Expect(err).NotTo(HaveOccurred()) - }, 5) + }) It("should return an error if the directory doesn't exist", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{invalidDirectory}, ErrorIfPathMissing: true, }) Expect(err).To(HaveOccurred()) - }, 5) + }) It("should return an error if the file doesn't exist", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{Paths: []string{ filepath.Join(".", "testdata", "fake.yaml")}, ErrorIfPathMissing: true, }) Expect(err).To(HaveOccurred()) - }, 5) + }) It("should return an error if the resource group version isn't found", func() { // Wait for a CRD where the Group and Version don't exist @@ -374,7 +372,7 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).To(HaveOccurred()) - }, 5) + }) It("should return an error if the resource isn't found in the group version", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ @@ -421,9 +419,9 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).To(HaveOccurred()) - }, 5) + }) - It("should reinstall the CRDs if already present in the cluster", func() { + It("should reinstall the CRDs if already present in the cluster", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata")}, @@ -433,27 +431,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -562,27 +560,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -680,10 +678,18 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) - }, 5) + }) }) - It("should update CRDs if already present in the cluster", func() { + It("should set a working KubeConfig", func(ctx SpecContext) { + kubeconfigRESTConfig, err := clientcmd.RESTConfigFromKubeConfig(env.KubeConfig) + Expect(err).ToNot(HaveOccurred()) + kubeconfigClient, err := client.New(kubeconfigRESTConfig, client.Options{Scheme: s}) + Expect(err).NotTo(HaveOccurred()) + Expect(kubeconfigClient.List(ctx, &apiextensionsv1.CustomResourceDefinitionList{})).To(Succeed()) + }) + + It("should update CRDs if already present in the cluster", func(ctx SpecContext) { // Install only the CRDv1 multi-version example crds, err = InstallCRDs(env.Config, CRDInstallOptions{ @@ -694,7 +700,7 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) Expect(len(crd.Spec.Versions)).To(BeEquivalentTo(2)) @@ -736,7 +742,7 @@ var _ = Describe("Test", func() { // Expect to find updated CRD crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) Expect(len(crd.Spec.Versions)).To(BeEquivalentTo(3)) @@ -771,11 +777,10 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) - }, 5) + }) Describe("UninstallCRDs", func() { - It("should uninstall the CRDs from the cluster", func() { - + It("should uninstall the CRDs from the cluster", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{validDirectory}, }) @@ -784,27 +789,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -920,7 +925,7 @@ var _ = Describe("Test", func() { placeholder := &apiextensionsv1.CustomResourceDefinition{} Eventually(func() bool { for _, crd := range crds { - err = c.Get(context.TODO(), types.NamespacedName{Name: crd}, placeholder) + err = c.Get(ctx, types.NamespacedName{Name: crd}, placeholder) notFound := err != nil && apierrors.IsNotFound(err) if !notFound { return false @@ -928,7 +933,7 @@ var _ = Describe("Test", func() { } return true }, 20).Should(BeTrue()) - }, 30) + }) }) Describe("Start", func() { @@ -937,14 +942,14 @@ var _ = Describe("Test", func() { _, err := env.Start() Expect(err).To(HaveOccurred()) Expect(env.Stop()).To(Succeed()) - }, 30) + }) It("should not raise an error on invalid dir when flag is disabled", func() { env := &Environment{ErrorIfCRDPathMissing: false, CRDDirectoryPaths: []string{invalidDirectory}} _, err := env.Start() Expect(err).NotTo(HaveOccurred()) Expect(env.Stop()).To(Succeed()) - }, 30) + }) }) Describe("Stop", func() { @@ -956,6 +961,6 @@ var _ = Describe("Test", func() { // check if the /tmp/envtest-serving-certs-* dir doesnt exists any more Expect(env.WebhookInstallOptions.LocalServingCertDir).ShouldNot(BeADirectory()) - }, 30) + }) }) }) diff --git a/pkg/envtest/komega/OWNERS b/pkg/envtest/komega/OWNERS new file mode 100644 index 0000000000..45f63b0e2e --- /dev/null +++ b/pkg/envtest/komega/OWNERS @@ -0,0 +1,13 @@ +approvers: + - controller-runtime-admins + - controller-runtime-maintainers + - controller-runtime-approvers + - schrej + - JoelSpeed +reviewers: + - controller-runtime-admins + - controller-runtime-maintainers + - controller-runtime-approvers + - controller-runtime-reviewers + - schrej + - JoelSpeed diff --git a/pkg/envtest/komega/default.go b/pkg/envtest/komega/default.go new file mode 100644 index 0000000000..dad1f551ae --- /dev/null +++ b/pkg/envtest/komega/default.go @@ -0,0 +1,102 @@ +package komega + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// defaultK is the Komega used by the package global functions. +var defaultK = &komega{ctx: context.Background()} + +// SetClient sets the client used by the package global functions. +func SetClient(c client.Client) { + defaultK.client = c +} + +// SetContext sets the context used by the package global functions. +func SetContext(c context.Context) { + defaultK.ctx = c +} + +func checkDefaultClient() { + if defaultK.client == nil { + panic("Default Komega's client is not set. Use SetClient to set it.") + } +} + +// Get returns a function that fetches a resource and returns the occurring error. +// It can be used with gomega.Eventually() like this +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(komega.Get(&deployment)).Should(gomega.Succeed()) +// +// By calling the returned function directly it can also be used with gomega.Expect(komega.Get(...)()).To(...) +func Get(obj client.Object) func() error { + checkDefaultClient() + return defaultK.Get(obj) +} + +// List returns a function that lists resources and returns the occurring error. +// It can be used with gomega.Eventually() like this +// +// deployments := v1.DeploymentList{ ... } +// gomega.Eventually(k.List(&deployments)).Should(gomega.Succeed()) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.List(...)()).To(...) +func List(list client.ObjectList, opts ...client.ListOption) func() error { + checkDefaultClient() + return defaultK.List(list, opts...) +} + +// Update returns a function that fetches a resource, applies the provided update function and then updates the resource. +// It can be used with gomega.Eventually() like this: +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(k.Update(&deployment, func() { +// deployment.Spec.Replicas = 3 +// })).Should(gomega.Succeed()) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.Update(...)()).To(...) +func Update(obj client.Object, f func(), opts ...client.UpdateOption) func() error { + checkDefaultClient() + return defaultK.Update(obj, f, opts...) +} + +// UpdateStatus returns a function that fetches a resource, applies the provided update function and then updates the resource's status. +// It can be used with gomega.Eventually() like this: +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(k.UpdateStatus(&deployment, func() { +// deployment.Status.AvailableReplicas = 1 +// })).Should(gomega.Succeed()) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.UpdateStatus(...)()).To(...) +func UpdateStatus(obj client.Object, f func(), opts ...client.SubResourceUpdateOption) func() error { + checkDefaultClient() + return defaultK.UpdateStatus(obj, f, opts...) +} + +// Object returns a function that fetches a resource and returns the object. +// It can be used with gomega.Eventually() like this: +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(k.Object(&deployment)).Should(HaveField("Spec.Replicas", gomega.Equal(ptr.To(3)))) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.Object(...)()).To(...) +func Object(obj client.Object) func() (client.Object, error) { + checkDefaultClient() + return defaultK.Object(obj) +} + +// ObjectList returns a function that fetches a resource and returns the object. +// It can be used with gomega.Eventually() like this: +// +// deployments := appsv1.DeploymentList{ ... } +// gomega.Eventually(k.ObjectList(&deployments)).Should(HaveField("Items", HaveLen(1))) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.ObjectList(...)()).To(...) +func ObjectList(list client.ObjectList, opts ...client.ListOption) func() (client.ObjectList, error) { + checkDefaultClient() + return defaultK.ObjectList(list, opts...) +} diff --git a/pkg/envtest/komega/default_test.go b/pkg/envtest/komega/default_test.go new file mode 100644 index 0000000000..1a1de72cf3 --- /dev/null +++ b/pkg/envtest/komega/default_test.go @@ -0,0 +1,116 @@ +package komega + +import ( + "testing" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestDefaultGet(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(Get(&fetched)).Should(Succeed()) + + g.Expect(*fetched.Spec.Replicas).To(BeEquivalentTo(5)) +} + +func TestDefaultList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + list := appsv1.DeploymentList{} + g.Eventually(List(&list)).Should(Succeed()) + + g.Expect(list.Items).To(HaveLen(1)) + depl := exampleDeployment() + g.Expect(list.Items[0]).To(And( + HaveField("ObjectMeta.Name", Equal(depl.ObjectMeta.Name)), + HaveField("ObjectMeta.Namespace", Equal(depl.ObjectMeta.Namespace)), + )) +} + +func TestDefaultUpdate(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(Update(&updateDeployment, func() { + updateDeployment.Annotations = map[string]string{"updated": "true"} + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(Object(&fetched)()).To(HaveField("ObjectMeta.Annotations", HaveKeyWithValue("updated", "true"))) +} + +func TestDefaultUpdateStatus(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(UpdateStatus(&updateDeployment, func() { + updateDeployment.Status.AvailableReplicas = 1 + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(Object(&fetched)()).To(HaveField("Status.AvailableReplicas", BeEquivalentTo(1))) +} + +func TestDefaultObject(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(Object(&fetched)).Should(And( + Not(BeNil()), + HaveField("Spec.Replicas", Equal(ptr.To(int32(5)))), + )) +} + +func TestDefaultObjectList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + list := appsv1.DeploymentList{} + g.Eventually(ObjectList(&list)).Should(And( + Not(BeNil()), + HaveField("Items", And( + HaveLen(1), + ContainElement(HaveField("Spec.Replicas", Equal(ptr.To(int32(5))))), + )), + )) +} diff --git a/pkg/envtest/komega/equalobject.go b/pkg/envtest/komega/equalobject.go new file mode 100644 index 0000000000..a931c2718a --- /dev/null +++ b/pkg/envtest/komega/equalobject.go @@ -0,0 +1,297 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package komega + +import ( + "fmt" + "reflect" + "strings" + + "github.com/google/go-cmp/cmp" + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/types" + "k8s.io/apimachinery/pkg/runtime" +) + +// These package variables hold pre-created commonly used options that can be used to reduce the manual work involved in +// identifying the paths that need to be compared for testing equality between objects. +var ( + // IgnoreAutogeneratedMetadata contains the paths for all the metadata fields that are commonly set by the + // client and APIServer. This is used as a MatchOption for situations when only user-provided metadata is relevant. + IgnoreAutogeneratedMetadata = IgnorePaths{ + "metadata.uid", + "metadata.generation", + "metadata.creationTimestamp", + "metadata.resourceVersion", + "metadata.managedFields", + "metadata.deletionGracePeriodSeconds", + "metadata.deletionTimestamp", + "metadata.selfLink", + "metadata.generateName", + } +) + +type diffPath struct { + types []string + json []string +} + +// equalObjectMatcher is a Gomega matcher used to establish equality between two Kubernetes runtime.Objects. +type equalObjectMatcher struct { + // original holds the object that will be used to Match. + original runtime.Object + + // diffPaths contains the paths that differ between two objects. + diffPaths []diffPath + + // options holds the options that identify what should and should not be matched. + options *EqualObjectOptions +} + +// EqualObject returns a Matcher for the passed Kubernetes runtime.Object with the passed Options. This function can be +// used as a Gomega Matcher in Gomega Assertions. +func EqualObject(original runtime.Object, opts ...EqualObjectOption) types.GomegaMatcher { + matchOptions := &EqualObjectOptions{} + matchOptions = matchOptions.ApplyOptions(opts) + + return &equalObjectMatcher{ + options: matchOptions, + original: original, + } +} + +// Match compares the current object to the passed object and returns true if the objects are the same according to +// the Matcher and MatchOptions. +func (m *equalObjectMatcher) Match(actual interface{}) (success bool, err error) { + // Nil checks required first here for: + // 1) Nil equality which returns true + // 2) One object nil which returns an error + actualIsNil := reflect.ValueOf(actual).IsNil() + originalIsNil := reflect.ValueOf(m.original).IsNil() + + if actualIsNil && originalIsNil { + return true, nil + } + if actualIsNil || originalIsNil { + return false, fmt.Errorf("can not compare an object with a nil. original %v , actual %v", m.original, actual) + } + + m.diffPaths = m.calculateDiff(actual) + return len(m.diffPaths) == 0, nil +} + +// FailureMessage returns a message comparing the full objects after an unexpected failure to match has occurred. +func (m *equalObjectMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("the following fields were expected to match but did not:\n%v\n%s", m.diffPaths, + format.Message(actual, "expected to match", m.original)) +} + +// NegatedFailureMessage returns a string stating that all fields matched, even though that was not expected. +func (m *equalObjectMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return "it was expected that some fields do not match, but all of them did" +} + +func (d diffPath) String() string { + return fmt.Sprintf("(%s/%s)", strings.Join(d.types, "."), strings.Join(d.json, ".")) +} + +// diffReporter is a custom recorder for cmp.Diff which records all paths that are +// different between two objects. +type diffReporter struct { + stack []cmp.PathStep + + diffPaths []diffPath +} + +func (r *diffReporter) PushStep(s cmp.PathStep) { + r.stack = append(r.stack, s) +} + +func (r *diffReporter) Report(res cmp.Result) { + if !res.Equal() { + r.diffPaths = append(r.diffPaths, r.currentPath()) + } +} + +// currentPath converts the current stack into string representations that match +// the IgnorePaths and MatchPaths syntax. +func (r *diffReporter) currentPath() diffPath { + p := diffPath{types: []string{""}, json: []string{""}} + for si, s := range r.stack[1:] { + switch s := s.(type) { + case cmp.StructField: + p.types = append(p.types, s.String()[1:]) + // fetch the type information from the parent struct. + // Note: si has an offset of 1 compared to r.stack as we loop over r.stack[1:], so we don't need -1 + field := r.stack[si].Type().Field(s.Index()) + p.json = append(p.json, strings.Split(field.Tag.Get("json"), ",")[0]) + case cmp.SliceIndex: + key := fmt.Sprintf("[%d]", s.Key()) + p.types[len(p.types)-1] += key + p.json[len(p.json)-1] += key + case cmp.MapIndex: + key := fmt.Sprintf("%v", s.Key()) + if strings.ContainsAny(key, ".[]/\\") { + key = fmt.Sprintf("[%s]", key) + p.types[len(p.types)-1] += key + p.json[len(p.json)-1] += key + } else { + p.types = append(p.types, key) + p.json = append(p.json, key) + } + } + } + // Empty strings were added as the first element. If they're still empty, remove them again. + if len(p.json) > 0 && len(p.json[0]) == 0 { + p.json = p.json[1:] + p.types = p.types[1:] + } + return p +} + +func (r *diffReporter) PopStep() { + r.stack = r.stack[:len(r.stack)-1] +} + +// calculateDiff calculates the difference between two objects and returns the +// paths of the fields that do not match. +func (m *equalObjectMatcher) calculateDiff(actual interface{}) []diffPath { + var original interface{} = m.original + // Remove the wrapping Object from unstructured.Unstructured to make comparison behave similar to + // regular objects. + if u, isUnstructured := actual.(runtime.Unstructured); isUnstructured { + actual = u.UnstructuredContent() + } + if u, ok := m.original.(runtime.Unstructured); ok { + original = u.UnstructuredContent() + } + r := diffReporter{} + cmp.Diff(original, actual, cmp.Reporter(&r)) + return filterDiffPaths(*m.options, r.diffPaths) +} + +// filterDiffPaths filters the diff paths using the paths in EqualObjectOptions. +func filterDiffPaths(opts EqualObjectOptions, paths []diffPath) []diffPath { + result := []diffPath{} + + for _, p := range paths { + if len(opts.matchPaths) > 0 && !hasAnyPathPrefix(p, opts.matchPaths) { + continue + } + if hasAnyPathPrefix(p, opts.ignorePaths) { + continue + } + + result = append(result, p) + } + + return result +} + +// hasPathPrefix compares the segments of a path. +func hasPathPrefix(path []string, prefix []string) bool { + for i, p := range prefix { + if i >= len(path) { + return false + } + // return false if a segment doesn't match + if path[i] != p && (i < len(prefix)-1 || !segmentHasPrefix(path[i], p)) { + return false + } + } + return true +} + +func segmentHasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[0:len(prefix)] == prefix && + // if it is a prefix match, make sure the next character is a [ for array/map access + (len(s) == len(prefix) || s[len(prefix)] == '[') +} + +// hasAnyPathPrefix returns true if path matches any of the path prefixes. +// It respects the name boundaries within paths, so 'ObjectMeta.Name' does not +// match 'ObjectMeta.Namespace' for example. +func hasAnyPathPrefix(path diffPath, prefixes [][]string) bool { + for _, prefix := range prefixes { + if hasPathPrefix(path.types, prefix) || hasPathPrefix(path.json, prefix) { + return true + } + } + return false +} + +// EqualObjectOption describes an Option that can be applied to a Matcher. +type EqualObjectOption interface { + // ApplyToEqualObjectMatcher applies this configuration to the given MatchOption. + ApplyToEqualObjectMatcher(options *EqualObjectOptions) +} + +// EqualObjectOptions holds the available types of EqualObjectOptions that can be applied to a Matcher. +type EqualObjectOptions struct { + ignorePaths [][]string + matchPaths [][]string +} + +// ApplyOptions adds the passed MatchOptions to the MatchOptions struct. +func (o *EqualObjectOptions) ApplyOptions(opts []EqualObjectOption) *EqualObjectOptions { + for _, opt := range opts { + opt.ApplyToEqualObjectMatcher(o) + } + return o +} + +// IgnorePaths instructs the Matcher to ignore given paths when computing a diff. +// Paths are written in a syntax similar to Go with a few special cases. Both types and +// json/yaml field names are supported. +// +// Regular Paths: +// * "ObjectMeta.Name" +// * "metadata.name" +// Arrays: +// * "metadata.ownerReferences[0].name" +// Maps, if they do not contain any of .[]/\: +// * "metadata.labels.something" +// Maps, if they contain any of .[]/\: +// * "metadata.labels[kubernetes.io/something]" +type IgnorePaths []string + +// ApplyToEqualObjectMatcher applies this configuration to the given MatchOptions. +func (i IgnorePaths) ApplyToEqualObjectMatcher(opts *EqualObjectOptions) { + for _, p := range i { + opts.ignorePaths = append(opts.ignorePaths, strings.Split(p, ".")) + } +} + +// MatchPaths instructs the Matcher to restrict its diff to the given paths. If empty the Matcher will look at all paths. +// Paths are written in a syntax similar to Go with a few special cases. Both types and +// json/yaml field names are supported. +// +// Regular Paths: +// * "ObjectMeta.Name" +// * "metadata.name" +// Arrays: +// * "metadata.ownerReferences[0].name" +// Maps, if they do not contain any of .[]/\: +// * "metadata.labels.something" +// Maps, if they contain any of .[]/\: +// * "metadata.labels[kubernetes.io/something]" +type MatchPaths []string + +// ApplyToEqualObjectMatcher applies this configuration to the given MatchOptions. +func (i MatchPaths) ApplyToEqualObjectMatcher(opts *EqualObjectOptions) { + for _, p := range i { + opts.matchPaths = append(opts.ignorePaths, strings.Split(p, ".")) + } +} diff --git a/pkg/envtest/komega/equalobject_test.go b/pkg/envtest/komega/equalobject_test.go new file mode 100644 index 0000000000..9fe10d1779 --- /dev/null +++ b/pkg/envtest/komega/equalobject_test.go @@ -0,0 +1,662 @@ +package komega + +import ( + "testing" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestEqualObjectMatcher(t *testing.T) { + cases := []struct { + name string + original client.Object + modified client.Object + options []EqualObjectOption + want bool + }{ + { + name: "succeed with equal objects", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + want: true, + }, + { + name: "fail with non equal objects", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somethingelse", + }, + }, + want: false, + }, + { + name: "succeeds if ignored fields do not match", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{"somelabel": "somevalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "controller", + }}, + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somethingelse", + Labels: map[string]string{"somelabel": "anothervalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "another", + }}, + }, + }, + want: true, + options: []EqualObjectOption{ + IgnorePaths{ + "ObjectMeta.Name", + "ObjectMeta.CreationTimestamp", + "ObjectMeta.Labels.somelabel", + "ObjectMeta.OwnerReferences[0].Name", + "Spec.Template.ObjectMeta", + }, + }, + }, + { + name: "succeeds if ignored fields in json notation do not match", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{"somelabel": "somevalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "controller", + }}, + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somethingelse", + Labels: map[string]string{"somelabel": "anothervalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "another", + }}, + }, + }, + want: true, + options: []EqualObjectOption{ + IgnorePaths{ + "metadata.name", + "metadata.creationTimestamp", + "metadata.labels.somelabel", + "metadata.ownerReferences[0].name", + "spec.template.metadata", + }, + }, + }, + { + name: "succeeds if all allowed fields match, and some others do not", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "special", + }, + }, + want: true, + options: []EqualObjectOption{ + MatchPaths{ + "ObjectMeta.Name", + }, + }, + }, + { + name: "works with unstructured.Unstructured", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "something", + "namespace": "test", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "somethingelse", + "namespace": "test", + }, + }, + }, + want: true, + options: []EqualObjectOption{ + IgnorePaths{ + "metadata.name", + }, + }, + }, + + // Test when objects are equal. + { + name: "Equal field (spec) both in original and in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: true, + }, + + { + name: "Equal nested field both in original and in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + want: true, + }, + + // Test when there is a difference between the objects. + { + name: "Unequal field both in original and in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar-changed", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: false, + }, + { + name: "Unequal nested field both in original and modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A-Changed", + }, + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + want: false, + }, + + { + name: "Value of type map with different values", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "map": map[string]string{ + "A": "A-changed", + "B": "B", + // C missing + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "map": map[string]string{ + "A": "A", + // B missing + "C": "C", + }, + }, + }, + }, + want: false, + }, + + { + name: "Value of type Array or Slice with same length but different values", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "slice": []string{ + "D", + "C", + "B", + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "slice": []string{ + "A", + "B", + "C", + }, + }, + }, + }, + want: false, + }, + + // This tests specific behaviour in how Kubernetes marshals the zero value of metav1.Time{}. + { + name: "Creation timestamp set to empty value on both original and modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": metav1.Time{}, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": metav1.Time{}, + }, + }, + }, + want: true, + }, + + // Cases to test diff when fields exist only in modified object. + { + name: "Field only in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: false, + }, + { + name: "Nested field only in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + want: false, + }, + { + name: "Creation timestamp exists on modified but not on original", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": "2021-11-03T11:05:17Z", + }, + }, + }, + want: false, + }, + + // Test when fields exists only in the original object. + { + name: "Field only in original", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + want: false, + }, + { + name: "Nested field only in original", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + want: false, + }, + { + name: "Creation timestamp exists on original but not on modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": "2021-11-03T11:05:17Z", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + + want: false, + }, + + // Test metadata fields computed by the system or in status are compared. + { + name: "Unequal Metadata fields computed by the system or in status", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "selfLink": "foo", + "uid": "foo", + "resourceVersion": "foo", + "generation": "foo", + "managedFields": "foo", + }, + "status": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: false, + }, + { + name: "Unequal labels and annotations", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "foo": "bar", + }, + "annotations": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + want: false, + }, + + // Ignore fields MatchOption + { + name: "Unequal metadata fields ignored by IgnorePaths MatchOption", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + "selfLink": "foo", + "uid": "foo", + "resourceVersion": "foo", + "generation": "foo", + "managedFields": "foo", + }, + }, + }, + options: []EqualObjectOption{IgnoreAutogeneratedMetadata}, + want: true, + }, + { + name: "Unequal labels and annotations ignored by IgnorePaths MatchOption", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + "labels": map[string]interface{}{ + "foo": "bar", + }, + "annotations": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + options: []EqualObjectOption{IgnorePaths{"metadata.labels", "metadata.annotations"}}, + want: true, + }, + { + name: "Ignore fields are not compared", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{}, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "controlPlaneEndpoint": map[string]interface{}{ + "host": "", + "port": 0, + }, + }, + }, + }, + options: []EqualObjectOption{IgnorePaths{"spec.controlPlaneEndpoint"}}, + want: true, + }, + { + name: "Not-ignored fields are still compared", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{}, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + "ignored": "somevalue", + "superflous": "shouldcausefailure", + }, + }, + }, + }, + options: []EqualObjectOption{IgnorePaths{"metadata.annotations.ignored"}}, + want: false, + }, + + // MatchPaths MatchOption + { + name: "Unequal metadata fields not compared by setting MatchPaths MatchOption", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "uid": "foo", + }, + }, + }, + options: []EqualObjectOption{MatchPaths{"spec"}}, + want: true, + }, + + // More tests + { + name: "No changes", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + "B": "B", + "C": "C", // C only in original + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + "B": "B", + }, + }, + }, + want: false, + }, + { + name: "Many changes", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + // B missing + "C": "C", // C only in original + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + "B": "B", + }, + }, + }, + want: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + g := NewWithT(t) + m := EqualObject(c.original, c.options...) + success, _ := m.Match(c.modified) + if !success { + t.Log(m.FailureMessage(c.modified)) + } + g.Expect(success).To(Equal(c.want)) + }) + } +} diff --git a/pkg/envtest/komega/interfaces.go b/pkg/envtest/komega/interfaces.go new file mode 100644 index 0000000000..b412e5c1bf --- /dev/null +++ b/pkg/envtest/komega/interfaces.go @@ -0,0 +1,76 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package komega + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Komega is a collection of utilites for writing tests involving a mocked +// Kubernetes API. +type Komega interface { + // Get returns a function that fetches a resource and returns the occurring error. + // It can be used with gomega.Eventually() like this + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Get(&deployment)).To(gomega.Succeed()) + // By calling the returned function directly it can also be used with gomega.Expect(k.Get(...)()).To(...) + Get(client.Object) func() error + + // List returns a function that lists resources and returns the occurring error. + // It can be used with gomega.Eventually() like this + // deployments := v1.DeploymentList{ ... } + // gomega.Eventually(k.List(&deployments)).To(gomega.Succeed()) + // By calling the returned function directly it can also be used as gomega.Expect(k.List(...)()).To(...) + List(client.ObjectList, ...client.ListOption) func() error + + // Update returns a function that fetches a resource, applies the provided update function and then updates the resource. + // It can be used with gomega.Eventually() like this: + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Update(&deployment, func() { + // deployment.Spec.Replicas = 3 + // })).To(gomega.Succeed()) + // By calling the returned function directly it can also be used as gomega.Expect(k.Update(...)()).To(...) + Update(client.Object, func(), ...client.UpdateOption) func() error + + // UpdateStatus returns a function that fetches a resource, applies the provided update function and then updates the resource's status. + // It can be used with gomega.Eventually() like this: + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Update(&deployment, func() { + // deployment.Status.AvailableReplicas = 1 + // })).To(gomega.Succeed()) + // By calling the returned function directly it can also be used as gomega.Expect(k.UpdateStatus(...)()).To(...) + UpdateStatus(client.Object, func(), ...client.SubResourceUpdateOption) func() error + + // Object returns a function that fetches a resource and returns the object. + // It can be used with gomega.Eventually() like this: + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Object(&deployment)).To(HaveField("Spec.Replicas", gomega.Equal(ptr.To(int32(3))))) + // By calling the returned function directly it can also be used as gomega.Expect(k.Object(...)()).To(...) + Object(client.Object) func() (client.Object, error) + + // ObjectList returns a function that fetches a resource and returns the object. + // It can be used with gomega.Eventually() like this: + // deployments := appsv1.DeploymentList{ ... } + // gomega.Eventually(k.ObjectList(&deployments)).To(HaveField("Items", HaveLen(1))) + // By calling the returned function directly it can also be used as gomega.Expect(k.ObjectList(...)()).To(...) + ObjectList(client.ObjectList, ...client.ListOption) func() (client.ObjectList, error) + + // WithContext returns a copy that uses the given context. + WithContext(context.Context) Komega +} diff --git a/pkg/envtest/komega/komega.go b/pkg/envtest/komega/komega.go new file mode 100644 index 0000000000..e19d9b5f0b --- /dev/null +++ b/pkg/envtest/komega/komega.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package komega + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// komega is a collection of utilites for writing tests involving a mocked +// Kubernetes API. +type komega struct { + ctx context.Context + client client.Client +} + +var _ Komega = &komega{} + +// New creates a new Komega instance with the given client. +func New(c client.Client) Komega { + return &komega{ + client: c, + ctx: context.Background(), + } +} + +// WithContext returns a copy that uses the given context. +func (k komega) WithContext(ctx context.Context) Komega { + k.ctx = ctx + return &k +} + +// Get returns a function that fetches a resource and returns the occurring error. +func (k *komega) Get(obj client.Object) func() error { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() error { + return k.client.Get(k.ctx, key, obj) + } +} + +// List returns a function that lists resources and returns the occurring error. +func (k *komega) List(obj client.ObjectList, opts ...client.ListOption) func() error { + return func() error { + return k.client.List(k.ctx, obj, opts...) + } +} + +// Update returns a function that fetches a resource, applies the provided update function and then updates the resource. +func (k *komega) Update(obj client.Object, updateFunc func(), opts ...client.UpdateOption) func() error { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() error { + err := k.client.Get(k.ctx, key, obj) + if err != nil { + return err + } + updateFunc() + return k.client.Update(k.ctx, obj, opts...) + } +} + +// UpdateStatus returns a function that fetches a resource, applies the provided update function and then updates the resource's status. +func (k *komega) UpdateStatus(obj client.Object, updateFunc func(), opts ...client.SubResourceUpdateOption) func() error { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() error { + err := k.client.Get(k.ctx, key, obj) + if err != nil { + return err + } + updateFunc() + return k.client.Status().Update(k.ctx, obj, opts...) + } +} + +// Object returns a function that fetches a resource and returns the object. +func (k *komega) Object(obj client.Object) func() (client.Object, error) { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() (client.Object, error) { + err := k.client.Get(k.ctx, key, obj) + return obj, err + } +} + +// ObjectList returns a function that fetches a resource and returns the object. +func (k *komega) ObjectList(obj client.ObjectList, opts ...client.ListOption) func() (client.ObjectList, error) { + return func() (client.ObjectList, error) { + err := k.client.List(k.ctx, obj, opts...) + return obj, err + } +} diff --git a/pkg/envtest/komega/komega_test.go b/pkg/envtest/komega/komega_test.go new file mode 100644 index 0000000000..8867ac239a --- /dev/null +++ b/pkg/envtest/komega/komega_test.go @@ -0,0 +1,138 @@ +package komega + +import ( + "testing" + + _ "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func exampleDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To(int32(5)), + }, + } +} + +func createFakeClient() client.Client { + return fakeclient.NewClientBuilder(). + WithObjects(exampleDeployment()). + Build() +} + +func TestGet(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(k.Get(&fetched)).Should(Succeed()) + + g.Expect(*fetched.Spec.Replicas).To(BeEquivalentTo(5)) +} + +func TestList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + list := appsv1.DeploymentList{} + g.Eventually(k.List(&list)).Should(Succeed()) + + g.Expect(list.Items).To(HaveLen(1)) + depl := exampleDeployment() + g.Expect(list.Items[0]).To(And( + HaveField("ObjectMeta.Name", Equal(depl.ObjectMeta.Name)), + HaveField("ObjectMeta.Namespace", Equal(depl.ObjectMeta.Namespace)), + )) +} + +func TestUpdate(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(k.Update(&updateDeployment, func() { + updateDeployment.Annotations = map[string]string{"updated": "true"} + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(k.Object(&fetched)()).To(HaveField("ObjectMeta.Annotations", HaveKeyWithValue("updated", "true"))) +} + +func TestUpdateStatus(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(k.UpdateStatus(&updateDeployment, func() { + updateDeployment.Status.AvailableReplicas = 1 + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(k.Object(&fetched)()).To(HaveField("Status.AvailableReplicas", BeEquivalentTo(1))) +} + +func TestObject(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(k.Object(&fetched)).Should(And( + Not(BeNil()), + HaveField("Spec.Replicas", Equal(ptr.To(int32(5)))), + )) +} + +func TestObjectList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + list := appsv1.DeploymentList{} + g.Eventually(k.ObjectList(&list)).Should(And( + Not(BeNil()), + HaveField("Items", And( + HaveLen(1), + ContainElement(HaveField("Spec.Replicas", Equal(ptr.To(int32(5))))), + )), + )) +} diff --git a/pkg/envtest/printer/ginkgo.go b/pkg/envtest/printer/ginkgo.go deleted file mode 100644 index d835dc7721..0000000000 --- a/pkg/envtest/printer/ginkgo.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package printer contains setup for a friendlier Ginkgo printer that's easier -// to parse by test automation. -package printer - -import ( - "fmt" - - "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/types" -) - -var _ ginkgo.Reporter = NewlineReporter{} - -// NewlineReporter is Reporter that Prints a newline after the default Reporter output so that the results -// are correctly parsed by test automation. -// See issue https://github.com/jstemmer/go-junit-report/issues/31 -type NewlineReporter struct{} - -// SpecSuiteWillBegin implements ginkgo.Reporter. -func (NewlineReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { -} - -// BeforeSuiteDidRun implements ginkgo.Reporter. -func (NewlineReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {} - -// AfterSuiteDidRun implements ginkgo.Reporter. -func (NewlineReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {} - -// SpecWillRun implements ginkgo.Reporter. -func (NewlineReporter) SpecWillRun(specSummary *types.SpecSummary) {} - -// SpecDidComplete implements ginkgo.Reporter. -func (NewlineReporter) SpecDidComplete(specSummary *types.SpecSummary) {} - -// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:". -func (NewlineReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { fmt.Printf("\n") } diff --git a/pkg/envtest/printer/prow.go b/pkg/envtest/printer/prow.go deleted file mode 100644 index 2f4009aa03..0000000000 --- a/pkg/envtest/printer/prow.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package printer - -import ( - "fmt" - "os" - "path/filepath" - "sync" - - "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" - "github.com/onsi/ginkgo/types" - - "k8s.io/apimachinery/pkg/util/sets" -) - -var ( - allRegisteredSuites = sets.String{} - allRegisteredSuitesLock = &sync.Mutex{} -) - -type prowReporter struct { - junitReporter *reporters.JUnitReporter -} - -// NewProwReporter returns a prowReporter that will write out junit if running in Prow and do -// nothing otherwise. -// WARNING: It seems this does not always properly fail the test runs when there are failures, -// see https://github.com/onsi/ginkgo/issues/706 -// When using this you must make sure to grep for failures in your junit xmls and fail the run -// if there are any. -func NewProwReporter(suiteName string) ginkgo.Reporter { - allRegisteredSuitesLock.Lock() - if allRegisteredSuites.Has(suiteName) { - panic(fmt.Sprintf("Suite named %q registered more than once", suiteName)) - } - allRegisteredSuites.Insert(suiteName) - allRegisteredSuitesLock.Unlock() - - if os.Getenv("CI") == "" { - return &prowReporter{} - } - artifactsDir := os.Getenv("ARTIFACTS") - if artifactsDir == "" { - return &prowReporter{} - } - - path := filepath.Join(artifactsDir, fmt.Sprintf("junit_%s_%d.xml", suiteName, config.GinkgoConfig.ParallelNode)) - return &prowReporter{ - junitReporter: reporters.NewJUnitReporter(path), - } -} - -func (pr *prowReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecSuiteWillBegin(config, summary) - } -} - -// BeforeSuiteDidRun implements ginkgo.Reporter. -func (pr *prowReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) { - if pr.junitReporter != nil { - pr.junitReporter.BeforeSuiteDidRun(setupSummary) - } -} - -// AfterSuiteDidRun implements ginkgo.Reporter. -func (pr *prowReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) { - if pr.junitReporter != nil { - pr.junitReporter.AfterSuiteDidRun(setupSummary) - } -} - -// SpecWillRun implements ginkgo.Reporter. -func (pr *prowReporter) SpecWillRun(specSummary *types.SpecSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecWillRun(specSummary) - } -} - -// SpecDidComplete implements ginkgo.Reporter. -func (pr *prowReporter) SpecDidComplete(specSummary *types.SpecSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecDidComplete(specSummary) - } -} - -// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:". -func (pr *prowReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecSuiteDidEnd(summary) - } -} diff --git a/pkg/envtest/server.go b/pkg/envtest/server.go index 5347f074de..9bb81ed2ab 100644 --- a/pkg/envtest/server.go +++ b/pkg/envtest/server.go @@ -17,15 +17,20 @@ limitations under the License. package envtest import ( + "context" "fmt" "os" "strings" "time" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" @@ -37,15 +42,14 @@ var log = logf.RuntimeLog.WithName("test-env") /* It's possible to override some defaults, by setting the following environment variables: - USE_EXISTING_CLUSTER (boolean): if set to true, envtest will use an existing cluster - TEST_ASSET_KUBE_APISERVER (string): path to the api-server binary to use - TEST_ASSET_ETCD (string): path to the etcd binary to use - TEST_ASSET_KUBECTL (string): path to the kubectl binary to use - KUBEBUILDER_ASSETS (string): directory containing the binaries to use (api-server, etcd and kubectl). Defaults to /usr/local/kubebuilder/bin. - KUBEBUILDER_CONTROLPLANE_START_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. - KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. - KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT (boolean): if set to true, the control plane's stdout and stderr are attached to os.Stdout and os.Stderr - +* USE_EXISTING_CLUSTER (boolean): if set to true, envtest will use an existing cluster +* TEST_ASSET_KUBE_APISERVER (string): path to the api-server binary to use +* TEST_ASSET_ETCD (string): path to the etcd binary to use +* TEST_ASSET_KUBECTL (string): path to the kubectl binary to use +* KUBEBUILDER_ASSETS (string): directory containing the binaries to use (api-server, etcd and kubectl). Defaults to /usr/local/kubebuilder/bin. +* KUBEBUILDER_CONTROLPLANE_START_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. +* KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. +* KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT (boolean): if set to true, the control plane's stdout and stderr are attached to os.Stdout and os.Stderr */ const ( envUseExistingCluster = "USE_EXISTING_CLUSTER" @@ -122,6 +126,10 @@ type Environment struct { // loading. Config *rest.Config + // KubeConfig provides []byte of a kubeconfig file to talk to the apiserver + // It's automatically populated if not set based on the `Config` + KubeConfig []byte + // CRDInstallOptions are the options for installing CRDs. CRDInstallOptions CRDInstallOptions @@ -143,8 +151,22 @@ type Environment struct { // values are merged. CRDDirectoryPaths []string + // DownloadBinaryAssets indicates that the envtest binaries should be downloaded. + // If BinaryAssetsDirectory is also set, it is used to store the downloaded binaries, + // otherwise a tmp directory is created. + DownloadBinaryAssets bool + + // DownloadBinaryAssetsVersion is the version of envtest binaries to download. + // Defaults to the latest stable version (i.e. excluding alpha / beta / RC versions). + DownloadBinaryAssetsVersion string + + // DownloadBinaryAssetsIndexURL is the index used to discover envtest binaries to download. + // Defaults to https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml. + DownloadBinaryAssetsIndexURL string + // BinaryAssetsDirectory is the path where the binaries required for the envtest are // located in the local environment. This field can be overridden by setting KUBEBUILDER_ASSETS. + // Set this field to SetupEnvtestDefaultBinaryAssetsDirectory() to share binaries with setup-envtest. BinaryAssetsDirectory string // UseExistingCluster indicates that this environments should use an @@ -162,11 +184,6 @@ type Environment struct { // environment variable or 20 seconds if unspecified ControlPlaneStopTimeout time.Duration - // KubeAPIServerFlags is the set of flags passed while starting the api server. - // - // Deprecated: use ControlPlane.GetAPIServer().Configure() instead. - KubeAPIServerFlags []string - // AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr. // Enable this to get more visibility of the testing control plane. // It respect KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT environment variable. @@ -211,19 +228,7 @@ func (te *Environment) Start() (*rest.Config, error) { } } else { apiServer := te.ControlPlane.GetAPIServer() - if len(apiServer.Args) == 0 { //nolint:staticcheck - // pass these through separately from above in case something like - // AddUser defaults APIServer. - // - // TODO(directxman12): if/when we feel like making a bigger - // breaking change here, just make APIServer and Etcd non-pointers - // in ControlPlane. - - // NB(directxman12): we still pass these in so that things work if the - // user manually specifies them, but in most cases we expect them to - // be nil so that we use the new .Configure() logic. - apiServer.Args = te.KubeAPIServerFlags //nolint:staticcheck - } + if te.ControlPlane.Etcd == nil { te.ControlPlane.Etcd = &controlplane.Etcd{} } @@ -231,22 +236,36 @@ func (te *Environment) Start() (*rest.Config, error) { if os.Getenv(envAttachOutput) == "true" { te.AttachControlPlaneOutput = true } - if apiServer.Out == nil && te.AttachControlPlaneOutput { - apiServer.Out = os.Stdout - } - if apiServer.Err == nil && te.AttachControlPlaneOutput { - apiServer.Err = os.Stderr - } - if te.ControlPlane.Etcd.Out == nil && te.AttachControlPlaneOutput { - te.ControlPlane.Etcd.Out = os.Stdout - } - if te.ControlPlane.Etcd.Err == nil && te.AttachControlPlaneOutput { - te.ControlPlane.Etcd.Err = os.Stderr + if te.AttachControlPlaneOutput { + if apiServer.Out == nil { + apiServer.Out = os.Stdout + } + if apiServer.Err == nil { + apiServer.Err = os.Stderr + } + if te.ControlPlane.Etcd.Out == nil { + te.ControlPlane.Etcd.Out = os.Stdout + } + if te.ControlPlane.Etcd.Err == nil { + te.ControlPlane.Etcd.Err = os.Stderr + } } - apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory) - te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory) - te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory) + if te.DownloadBinaryAssets { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(context.TODO(), + te.BinaryAssetsDirectory, te.DownloadBinaryAssetsVersion, te.DownloadBinaryAssetsIndexURL) + if err != nil { + return nil, err + } + + apiServer.Path = apiServerPath + te.ControlPlane.Etcd.Path = etcdPath + te.ControlPlane.KubectlPath = kubectlPath + } else { + apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory) + te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory) + te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory) + } if err := te.defaultTimeouts(); err != nil { return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err) @@ -276,11 +295,25 @@ func (te *Environment) Start() (*rest.Config, error) { te.Config = adminUser.Config() } + if len(te.KubeConfig) == 0 { + var err error + te.KubeConfig, err = controlplane.KubeConfigFromREST(te.Config) + if err != nil { + return nil, fmt.Errorf("unable to set KubeConfig field: %w", err) + } + } + // Set the default scheme if nil. if te.Scheme == nil { te.Scheme = scheme.Scheme } + // If we are bringing etcd up for the first time, it can take some time for the + // default namespace to actually be created and seen as available to the apiserver + if err := te.waitForDefaultNamespace(te.Config); err != nil { + return nil, fmt.Errorf("default namespace didn't register within deadline: %w", err) + } + // Call PrepWithoutInstalling to setup certificates first // and have them available to patch CRD conversion webhook as well. if err := te.WebhookInstallOptions.PrepWithoutInstalling(); err != nil { @@ -288,6 +321,9 @@ func (te *Environment) Start() (*rest.Config, error) { } log.V(1).Info("installing CRDs") + if te.CRDInstallOptions.Scheme == nil { + te.CRDInstallOptions.Scheme = te.Scheme + } te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs) te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths) te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing @@ -335,6 +371,20 @@ func (te *Environment) startControlPlane() error { return nil } +func (te *Environment) waitForDefaultNamespace(config *rest.Config) error { + cs, err := client.New(config, client.Options{}) + if err != nil { + return fmt.Errorf("unable to create client: %w", err) + } + // It shouldn't take longer than 5s for the default namespace to be brought up in etcd + return wait.PollUntilContextTimeout(context.TODO(), time.Millisecond*50, time.Second*5, true, func(ctx context.Context) (bool, error) { + if err = cs.Get(ctx, types.NamespacedName{Name: "default"}, &corev1.Namespace{}); err != nil { + return false, nil //nolint:nilerr + } + return true, nil + }) +} + func (te *Environment) defaultTimeouts() error { var err error if te.ControlPlaneStartTimeout == 0 { diff --git a/pkg/envtest/webhook.go b/pkg/envtest/webhook.go index 8552d3ba61..a6961bf7c6 100644 --- a/pkg/envtest/webhook.go +++ b/pkg/envtest/webhook.go @@ -16,7 +16,6 @@ package envtest import ( "context" "fmt" - "io/ioutil" "net" "os" "path/filepath" @@ -50,6 +49,11 @@ type WebhookInstallOptions struct { // ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install ValidatingWebhooks []*admissionv1.ValidatingWebhookConfiguration + // IgnoreSchemeConvertible, will modify any CRD conversion webhook to use the local serving host and port, + // bypassing the need to have the types registered in the Scheme. This is useful for testing CRD conversion webhooks + // with unregistered or unstructured types. + IgnoreSchemeConvertible bool + // IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true IgnoreErrorIfPathMissing bool @@ -117,7 +121,7 @@ func (o *WebhookInstallOptions) generateHostPort() (string, error) { if o.LocalServingPort == 0 { port, host, err := addr.Suggest(o.LocalServingHost) if err != nil { - return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err) + return "", fmt.Errorf("unable to grab random port for serving webhooks on: %w", err) } o.LocalServingPort = port o.LocalServingHost = host @@ -148,6 +152,8 @@ func (o *WebhookInstallOptions) PrepWithoutInstalling() error { // Install installs specified webhooks to the API server. func (o *WebhookInstallOptions) Install(config *rest.Config) error { + defaultWebhookOptions(o) + if len(o.LocalServingCAData) == 0 { if err := o.PrepWithoutInstalling(); err != nil { return err @@ -169,22 +175,33 @@ func (o *WebhookInstallOptions) Cleanup() error { return nil } +// defaultWebhookOptions sets the default values for Webhooks. +func defaultWebhookOptions(o *WebhookInstallOptions) { + if o.MaxTime == 0 { + o.MaxTime = defaultMaxWait + } + if o.PollInterval == 0 { + o.PollInterval = defaultPollInterval + } +} + // WaitForWebhooks waits for the Webhooks to be available through API server. func WaitForWebhooks(config *rest.Config, mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration, validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration, - options WebhookInstallOptions) error { - waitingFor := map[schema.GroupVersionKind]*sets.String{} + options WebhookInstallOptions, +) error { + waitingFor := map[schema.GroupVersionKind]*sets.Set[string]{} for _, hook := range mutatingWebhooks { h := hook gvk, err := apiutil.GVKForObject(h, scheme.Scheme) if err != nil { - return fmt.Errorf("unable to get gvk for MutatingWebhookConfiguration %s: %v", hook.GetName(), err) + return fmt.Errorf("unable to get gvk for MutatingWebhookConfiguration %s: %w", hook.GetName(), err) } if _, ok := waitingFor[gvk]; !ok { - waitingFor[gvk] = &sets.String{} + waitingFor[gvk] = &sets.Set[string]{} } waitingFor[gvk].Insert(h.GetName()) } @@ -193,18 +210,18 @@ func WaitForWebhooks(config *rest.Config, h := hook gvk, err := apiutil.GVKForObject(h, scheme.Scheme) if err != nil { - return fmt.Errorf("unable to get gvk for ValidatingWebhookConfiguration %s: %v", hook.GetName(), err) + return fmt.Errorf("unable to get gvk for ValidatingWebhookConfiguration %s: %w", hook.GetName(), err) } if _, ok := waitingFor[gvk]; !ok { - waitingFor[gvk] = &sets.String{} + waitingFor[gvk] = &sets.Set[string]{} } waitingFor[gvk].Insert(hook.GetName()) } // Poll until all resources are found in discovery p := &webhookPoller{config: config, waitingFor: waitingFor} - return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll) + return wait.PollUntilContextTimeout(context.TODO(), options.PollInterval, options.MaxTime, true, p.poll) } // poller checks if all the resources have been found in discovery, and returns false if not. @@ -213,11 +230,11 @@ type webhookPoller struct { config *rest.Config // waitingFor is the map of resources keyed by group version that have not yet been found in discovery - waitingFor map[schema.GroupVersionKind]*sets.String + waitingFor map[schema.GroupVersionKind]*sets.Set[string] } // poll checks if all the resources have been found in discovery, and returns false if not. -func (p *webhookPoller) poll() (done bool, err error) { +func (p *webhookPoller) poll(ctx context.Context) (done bool, err error) { // Create a new clientset to avoid any client caching of discovery c, err := client.New(p.config, client.Options{}) if err != nil { @@ -230,8 +247,8 @@ func (p *webhookPoller) poll() (done bool, err error) { delete(p.waitingFor, gvk) continue } - for _, name := range names.List() { - var obj = &unstructured.Unstructured{} + for _, name := range names.UnsortedList() { + obj := &unstructured.Unstructured{} obj.SetGroupVersionKind(gvk) err := c.Get(context.Background(), client.ObjectKey{ Namespace: "", @@ -257,31 +274,31 @@ func (p *webhookPoller) poll() (done bool, err error) { func (o *WebhookInstallOptions) setupCA() error { hookCA, err := certs.NewTinyCA() if err != nil { - return fmt.Errorf("unable to set up webhook CA: %v", err) + return fmt.Errorf("unable to set up webhook CA: %w", err) } names := []string{"localhost", o.LocalServingHost, o.LocalServingHostExternalName} hookCert, err := hookCA.NewServingCert(names...) if err != nil { - return fmt.Errorf("unable to set up webhook serving certs: %v", err) + return fmt.Errorf("unable to set up webhook serving certs: %w", err) } - localServingCertsDir, err := ioutil.TempDir("", "envtest-serving-certs-") + localServingCertsDir, err := os.MkdirTemp("", "envtest-serving-certs-") o.LocalServingCertDir = localServingCertsDir if err != nil { - return fmt.Errorf("unable to create directory for webhook serving certs: %v", err) + return fmt.Errorf("unable to create directory for webhook serving certs: %w", err) } certData, keyData, err := hookCert.AsBytes() if err != nil { - return fmt.Errorf("unable to marshal webhook serving certs: %v", err) + return fmt.Errorf("unable to marshal webhook serving certs: %w", err) } - if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec - return fmt.Errorf("unable to write webhook serving cert to disk: %v", err) + if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { + return fmt.Errorf("unable to write webhook serving cert to disk: %w", err) } - if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec - return fmt.Errorf("unable to write webhook serving key to disk: %v", err) + if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { + return fmt.Errorf("unable to write webhook serving key to disk: %w", err) } o.LocalServingCAData = certData @@ -296,14 +313,12 @@ func createWebhooks(config *rest.Config, mutHooks []*admissionv1.MutatingWebhook // Create each webhook for _, hook := range mutHooks { - hook := hook log.V(1).Info("installing mutating webhook", "webhook", hook.GetName()) if err := ensureCreated(cs, hook); err != nil { return err } } for _, hook := range valHooks { - hook := hook log.V(1).Info("installing validating webhook", "webhook", hook.GetName()) if err := ensureCreated(cs, hook); err != nil { return err @@ -359,7 +374,7 @@ func parseWebhook(options *WebhookInstallOptions) error { // returns slice of mutating and validating webhook configurations. func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []*admissionv1.ValidatingWebhookConfiguration, error) { // Get the webhook files - var files []os.FileInfo + var files []string var err error log.V(1).Info("reading Webhooks from path", "path", path) info, err := os.Stat(path) @@ -367,9 +382,15 @@ func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []* return nil, nil, err } if !info.IsDir() { - path, files = filepath.Dir(path), []os.FileInfo{info} - } else if files, err = ioutil.ReadDir(path); err != nil { - return nil, nil, err + path, files = filepath.Dir(path), []string{info.Name()} + } else { + entries, err := os.ReadDir(path) + if err != nil { + return nil, nil, err + } + for _, e := range entries { + files = append(files, e.Name()) + } } // file extensions that may contain Webhooks @@ -379,12 +400,12 @@ func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []* var valHooks []*admissionv1.ValidatingWebhookConfiguration for _, file := range files { // Only parse allowlisted file types - if !resourceExtensions.Has(filepath.Ext(file.Name())) { + if !resourceExtensions.Has(filepath.Ext(file)) { continue } // Unmarshal Webhooks from file into structs - docs, err := readDocuments(filepath.Join(path, file.Name())) + docs, err := readDocuments(filepath.Join(path, file)) if err != nil { return nil, nil, err } @@ -398,8 +419,8 @@ func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []* const ( admissionregv1 = "admissionregistration.k8s.io/v1" ) - switch { - case generic.Kind == "MutatingWebhookConfiguration": + switch generic.Kind { + case "MutatingWebhookConfiguration": if generic.APIVersion != admissionregv1 { return nil, nil, fmt.Errorf("only v1 is supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name) } @@ -408,7 +429,7 @@ func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []* return nil, nil, err } mutHooks = append(mutHooks, hook) - case generic.Kind == "ValidatingWebhookConfiguration": + case "ValidatingWebhookConfiguration": if generic.APIVersion != admissionregv1 { return nil, nil, fmt.Errorf("only v1 is supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name) } @@ -422,7 +443,7 @@ func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []* } } - log.V(1).Info("read webhooks from file", "file", file.Name()) + log.V(1).Info("read webhooks from file", "file", file) } return mutHooks, valHooks, nil } diff --git a/pkg/envtest/webhook_test.go b/pkg/envtest/webhook_test.go index 6f900bb7fb..47550fa147 100644 --- a/pkg/envtest/webhook_test.go +++ b/pkg/envtest/webhook_test.go @@ -18,15 +18,18 @@ package envtest import ( "context" + "crypto/tls" "path/filepath" + "strings" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -34,19 +37,21 @@ import ( ) var _ = Describe("Test", func() { - Describe("Webhook", func() { - It("should reject create request for webhook that rejects all requests", func() { + It("should reject create request for webhook that rejects all requests", func(specCtx SpecContext) { m, err := manager.New(env.Config, manager.Options{ - Port: env.WebhookInstallOptions.LocalServingPort, - Host: env.WebhookInstallOptions.LocalServingHost, - CertDir: env.WebhookInstallOptions.LocalServingCertDir, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: env.WebhookInstallOptions.LocalServingPort, + Host: env.WebhookInstallOptions.LocalServingHost, + CertDir: env.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + }), }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { _ = server.Start(ctx) }() @@ -82,8 +87,8 @@ var _ = Describe("Test", func() { } Eventually(func() bool { - err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + err = c.Create(ctx, obj) + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) cancel() @@ -95,8 +100,8 @@ var _ = Describe("Test", func() { } err := parseWebhook(&installOptions) Expect(err).NotTo(HaveOccurred()) - Expect(len(installOptions.MutatingWebhooks)).To(Equal(2)) - Expect(len(installOptions.ValidatingWebhooks)).To(Equal(2)) + Expect(installOptions.MutatingWebhooks).To(HaveLen(2)) + Expect(installOptions.ValidatingWebhooks).To(HaveLen(2)) }) It("should load webhooks from files", func() { @@ -105,8 +110,8 @@ var _ = Describe("Test", func() { } err := parseWebhook(&installOptions) Expect(err).NotTo(HaveOccurred()) - Expect(len(installOptions.MutatingWebhooks)).To(Equal(2)) - Expect(len(installOptions.ValidatingWebhooks)).To(Equal(2)) + Expect(installOptions.MutatingWebhooks).To(HaveLen(2)) + Expect(installOptions.ValidatingWebhooks).To(HaveLen(2)) }) }) }) diff --git a/pkg/event/event.go b/pkg/event/event.go index 271b3c00fb..82b1793f53 100644 --- a/pkg/event/event.go +++ b/pkg/event/event.go @@ -18,38 +18,58 @@ package event import "sigs.k8s.io/controller-runtime/pkg/client" -// CreateEvent is an event where a Kubernetes object was created. CreateEvent should be generated +// CreateEvent is an event where a Kubernetes object was created. CreateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by a handler.EventHandler. +type CreateEvent = TypedCreateEvent[client.Object] + +// UpdateEvent is an event where a Kubernetes object was updated. UpdateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. +type UpdateEvent = TypedUpdateEvent[client.Object] + +// DeleteEvent is an event where a Kubernetes object was deleted. DeleteEvent should be generated // by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. -type CreateEvent struct { +type DeleteEvent = TypedDeleteEvent[client.Object] + +// GenericEvent is an event where the operation type is unknown (e.g. polling or event originating outside the cluster). +// GenericEvent should be generated by a source.Source and transformed into a reconcile.Request by an +// handler.EventHandler. +type GenericEvent = TypedGenericEvent[client.Object] + +// TypedCreateEvent is an event where a Kubernetes object was created. TypedCreateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.TypedEventHandler. +type TypedCreateEvent[object any] struct { // Object is the object from the event - Object client.Object + Object object + + // IsInInitialList is true if the Create event was triggered by the initial list. + IsInInitialList bool } -// UpdateEvent is an event where a Kubernetes object was updated. UpdateEvent should be generated -// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. -type UpdateEvent struct { +// TypedUpdateEvent is an event where a Kubernetes object was updated. TypedUpdateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.TypedEventHandler. +type TypedUpdateEvent[object any] struct { // ObjectOld is the object from the event - ObjectOld client.Object + ObjectOld object // ObjectNew is the object from the event - ObjectNew client.Object + ObjectNew object } -// DeleteEvent is an event where a Kubernetes object was deleted. DeleteEvent should be generated -// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. -type DeleteEvent struct { +// TypedDeleteEvent is an event where a Kubernetes object was deleted. TypedDeleteEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.TypedEventHandler. +type TypedDeleteEvent[object any] struct { // Object is the object from the event - Object client.Object + Object object // DeleteStateUnknown is true if the Delete event was missed but we identified the object // as having been deleted. DeleteStateUnknown bool } -// GenericEvent is an event where the operation type is unknown (e.g. polling or event originating outside the cluster). -// GenericEvent should be generated by a source.Source and transformed into a reconcile.Request by an -// handler.EventHandler. -type GenericEvent struct { +// TypedGenericEvent is an event where the operation type is unknown (e.g. polling or event originating outside the cluster). +// TypedGenericEvent should be generated by a source.Source and transformed into a reconcile.Request by an +// handler.TypedEventHandler. +type TypedGenericEvent[object any] struct { // Object is the object from the event - Object client.Object + Object object } diff --git a/pkg/finalizer/finalizer.go b/pkg/finalizer/finalizer.go index 1f627f8c49..10c5645dbe 100644 --- a/pkg/finalizer/finalizer.go +++ b/pkg/finalizer/finalizer.go @@ -64,7 +64,7 @@ func (f finalizers) Finalize(ctx context.Context, obj client.Object) (Result, er // object (e.g. it may set a condition and need a status update). res.Updated = res.Updated || finalizerRes.Updated res.StatusUpdated = res.StatusUpdated || finalizerRes.StatusUpdated - errList = append(errList, fmt.Errorf("finalizer %q failed: %v", key, err)) + errList = append(errList, fmt.Errorf("finalizer %q failed: %w", key, err)) } else { // If the finalizer succeeds, we remove the finalizer from the primary // object's metadata, so we know it will need an update. diff --git a/pkg/finalizer/finalizer_test.go b/pkg/finalizer/finalizer_test.go index 944acd595a..c6848f6473 100644 --- a/pkg/finalizer/finalizer_test.go +++ b/pkg/finalizer/finalizer_test.go @@ -5,12 +5,11 @@ import ( "fmt" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) type mockFinalizer struct { @@ -21,10 +20,10 @@ type mockFinalizer struct { func (f mockFinalizer) Finalize(context.Context, client.Object) (Result, error) { return f.result, f.err } + func TestFinalizer(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Finalizer Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Finalizer Suite") } var _ = Describe("TestFinalizer", func() { @@ -42,97 +41,97 @@ var _ = Describe("TestFinalizer", func() { Describe("Register", func() { It("successfully registers a finalizer", func() { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) }) It("should fail when trying to register a finalizer that was already registered", func() { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) // calling Register again with the same key should return an error err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) - Expect(err).NotTo(BeNil()) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("already registered")) }) }) Describe("Finalize", func() { - It("successfully finalizes and returns true for Updated when deletion timestamp is nil and finalizer does not exist", func() { + It("successfully finalizes and returns true for Updated when deletion timestamp is nil and finalizer does not exist", func(ctx SpecContext) { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) pod.DeletionTimestamp = nil pod.Finalizers = []string{} - result, err := finalizers.Finalize(context.TODO(), pod) - Expect(err).To(BeNil()) + result, err := finalizers.Finalize(ctx, pod) + Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) // when deletion timestamp is nil and finalizer is not present, the registered finalizer would be added to the obj - Expect(len(pod.Finalizers)).To(Equal(1)) + Expect(pod.Finalizers).To(HaveLen(1)) Expect(pod.Finalizers[0]).To(Equal("finalizers.sigs.k8s.io/testfinalizer")) }) - It("successfully finalizes and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func() { + It("successfully finalizes and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) pod.Finalizers = []string{"finalizers.sigs.k8s.io/testfinalizer"} - result, err := finalizers.Finalize(context.TODO(), pod) - Expect(err).To(BeNil()) + result, err := finalizers.Finalize(ctx, pod) + Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) // finalizer will be removed from the obj upon successful finalization - Expect(len(pod.Finalizers)).To(Equal(0)) + Expect(pod.Finalizers).To(BeEmpty()) }) - It("should return no error and return false for Updated when deletion timestamp is nil and finalizer doesn't exist", func() { + It("should return no error and return false for Updated when deletion timestamp is nil and finalizer doesn't exist", func(ctx SpecContext) { pod.DeletionTimestamp = nil pod.Finalizers = []string{} - result, err := finalizers.Finalize(context.TODO(), pod) - Expect(err).To(BeNil()) + result, err := finalizers.Finalize(ctx, pod) + Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeFalse()) - Expect(len(pod.Finalizers)).To(Equal(0)) + Expect(pod.Finalizers).To(BeEmpty()) }) - It("should return no error and return false for Updated when deletion timestamp is not nil and the finalizer doesn't exist", func() { + It("should return no error and return false for Updated when deletion timestamp is not nil and the finalizer doesn't exist", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now pod.Finalizers = []string{} - result, err := finalizers.Finalize(context.TODO(), pod) - Expect(err).To(BeNil()) + result, err := finalizers.Finalize(ctx, pod) + Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeFalse()) - Expect(len(pod.Finalizers)).To(Equal(0)) + Expect(pod.Finalizers).To(BeEmpty()) }) - It("successfully finalizes multiple finalizers and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func() { + It("successfully finalizes multiple finalizers and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) err = finalizers.Register("finalizers.sigs.k8s.io/newtestfinalizer", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) pod.Finalizers = []string{"finalizers.sigs.k8s.io/testfinalizer", "finalizers.sigs.k8s.io/newtestfinalizer"} - result, err := finalizers.Finalize(context.TODO(), pod) - Expect(err).To(BeNil()) + result, err := finalizers.Finalize(ctx, pod) + Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) Expect(result.StatusUpdated).To(BeFalse()) - Expect(len(pod.Finalizers)).To(Equal(0)) + Expect(pod.Finalizers).To(BeEmpty()) }) - It("should return result as false and a non-nil error", func() { + It("should return result as false and a non-nil error", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now pod.Finalizers = []string{"finalizers.sigs.k8s.io/testfinalizer"} @@ -142,18 +141,18 @@ var _ = Describe("TestFinalizer", func() { f.err = fmt.Errorf("finalizer failed for %q", pod.Finalizers[0]) err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) - result, err := finalizers.Finalize(context.TODO(), pod) - Expect(err).ToNot(BeNil()) + result, err := finalizers.Finalize(ctx, pod) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("finalizer failed")) Expect(result.Updated).To(BeFalse()) Expect(result.StatusUpdated).To(BeFalse()) - Expect(len(pod.Finalizers)).To(Equal(1)) + Expect(pod.Finalizers).To(HaveLen(1)) Expect(pod.Finalizers[0]).To(Equal("finalizers.sigs.k8s.io/testfinalizer")) }) - It("should return expected result values and error values when registering multiple finalizers", func() { + It("should return expected result values and error values when registering multiple finalizers", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now pod.Finalizers = []string{ @@ -168,15 +167,15 @@ var _ = Describe("TestFinalizer", func() { f.result.StatusUpdated = false f.err = nil err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer1", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) - result, err := finalizers.Finalize(context.TODO(), pod) - Expect(err).To(BeNil()) + result, err := finalizers.Finalize(ctx, pod) + Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) Expect(result.StatusUpdated).To(BeFalse()) // `finalizers.sigs.k8s.io/testfinalizer1` will be removed from the list // of finalizers, so length will be 2. - Expect(len(pod.Finalizers)).To(Equal(2)) + Expect(pod.Finalizers).To(HaveLen(2)) Expect(pod.Finalizers[0]).To(Equal("finalizers.sigs.k8s.io/testfinalizer2")) Expect(pod.Finalizers[1]).To(Equal("finalizers.sigs.k8s.io/testfinalizer3")) @@ -185,14 +184,14 @@ var _ = Describe("TestFinalizer", func() { f.result.StatusUpdated = false f.err = fmt.Errorf("finalizer failed") err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer2", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) - result, err = finalizers.Finalize(context.TODO(), pod) - Expect(err).ToNot(BeNil()) + result, err = finalizers.Finalize(ctx, pod) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("finalizer failed")) Expect(result.Updated).To(BeFalse()) Expect(result.StatusUpdated).To(BeFalse()) - Expect(len(pod.Finalizers)).To(Equal(2)) + Expect(pod.Finalizers).To(HaveLen(2)) Expect(pod.Finalizers[0]).To(Equal("finalizers.sigs.k8s.io/testfinalizer2")) Expect(pod.Finalizers[1]).To(Equal("finalizers.sigs.k8s.io/testfinalizer3")) @@ -201,14 +200,14 @@ var _ = Describe("TestFinalizer", func() { f.result.StatusUpdated = true f.err = fmt.Errorf("finalizer failed") err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer3", f) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) - result, err = finalizers.Finalize(context.TODO(), pod) - Expect(err).ToNot(BeNil()) + result, err = finalizers.Finalize(ctx, pod) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("finalizer failed")) Expect(result.Updated).To(BeTrue()) Expect(result.StatusUpdated).To(BeTrue()) - Expect(len(pod.Finalizers)).To(Equal(2)) + Expect(pod.Finalizers).To(HaveLen(2)) Expect(pod.Finalizers[0]).To(Equal("finalizers.sigs.k8s.io/testfinalizer2")) Expect(pod.Finalizers[1]).To(Equal("finalizers.sigs.k8s.io/testfinalizer3")) }) diff --git a/pkg/handler/doc.go b/pkg/handler/doc.go index 3b5b79048d..e5fd177aff 100644 --- a/pkg/handler/doc.go +++ b/pkg/handler/doc.go @@ -21,7 +21,7 @@ Controller.Watch in order to generate and enqueue reconcile.Request work items. Generally, following premade event handlers should be sufficient for most use cases: -EventHandlers +EventHandlers: EnqueueRequestForObject - Enqueues a reconcile.Request containing the Name and Namespace of the object in the Event. This will cause the object that was the source of the Event (e.g. the created / deleted / updated object) to be diff --git a/pkg/handler/enqueue.go b/pkg/handler/enqueue.go index e6d3a4eaab..64cbe8a4d1 100644 --- a/pkg/handler/enqueue.go +++ b/pkg/handler/enqueue.go @@ -17,8 +17,12 @@ limitations under the License. package handler import ( + "context" + "reflect" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -31,43 +35,57 @@ type empty struct{} var _ EventHandler = &EnqueueRequestForObject{} // EnqueueRequestForObject enqueues a Request containing the Name and Namespace of the object that is the source of the Event. -// (e.g. the created / deleted / updated objects Name and Namespace). handler.EnqueueRequestForObject is used by almost all +// (e.g. the created / deleted / updated objects Name and Namespace). handler.EnqueueRequestForObject is used by almost all +// Controllers that have associated Resources (e.g. CRDs) to reconcile the associated Resource. +type EnqueueRequestForObject = TypedEnqueueRequestForObject[client.Object] + +// TypedEnqueueRequestForObject enqueues a Request containing the Name and Namespace of the object that is the source of the Event. +// (e.g. the created / deleted / updated objects Name and Namespace). handler.TypedEnqueueRequestForObject is used by almost all // Controllers that have associated Resources (e.g. CRDs) to reconcile the associated Resource. -type EnqueueRequestForObject struct{} +// +// TypedEnqueueRequestForObject is experimental and subject to future change. +type TypedEnqueueRequestForObject[object client.Object] struct{} // Create implements EventHandler. -func (e *EnqueueRequestForObject) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { - if evt.Object == nil { +func (e *TypedEnqueueRequestForObject[T]) Create(ctx context.Context, evt event.TypedCreateEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if isNil(evt.Object) { enqueueLog.Error(nil, "CreateEvent received with no metadata", "event", evt) return } - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + + item := reconcile.Request{NamespacedName: types.NamespacedName{ Name: evt.Object.GetName(), Namespace: evt.Object.GetNamespace(), - }}) + }} + + addToQueueCreate(q, evt, item) } // Update implements EventHandler. -func (e *EnqueueRequestForObject) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *TypedEnqueueRequestForObject[T]) Update(ctx context.Context, evt event.TypedUpdateEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { switch { - case evt.ObjectNew != nil: - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + case !isNil(evt.ObjectNew): + item := reconcile.Request{NamespacedName: types.NamespacedName{ Name: evt.ObjectNew.GetName(), Namespace: evt.ObjectNew.GetNamespace(), - }}) - case evt.ObjectOld != nil: - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + }} + + addToQueueUpdate(q, evt, item) + case !isNil(evt.ObjectOld): + item := reconcile.Request{NamespacedName: types.NamespacedName{ Name: evt.ObjectOld.GetName(), Namespace: evt.ObjectOld.GetNamespace(), - }}) + }} + + addToQueueUpdate(q, evt, item) default: enqueueLog.Error(nil, "UpdateEvent received with no metadata", "event", evt) } } // Delete implements EventHandler. -func (e *EnqueueRequestForObject) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { - if evt.Object == nil { +func (e *TypedEnqueueRequestForObject[T]) Delete(ctx context.Context, evt event.TypedDeleteEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if isNil(evt.Object) { enqueueLog.Error(nil, "DeleteEvent received with no metadata", "event", evt) return } @@ -78,8 +96,8 @@ func (e *EnqueueRequestForObject) Delete(evt event.DeleteEvent, q workqueue.Rate } // Generic implements EventHandler. -func (e *EnqueueRequestForObject) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { - if evt.Object == nil { +func (e *TypedEnqueueRequestForObject[T]) Generic(ctx context.Context, evt event.TypedGenericEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if isNil(evt.Object) { enqueueLog.Error(nil, "GenericEvent received with no metadata", "event", evt) return } @@ -88,3 +106,15 @@ func (e *EnqueueRequestForObject) Generic(evt event.GenericEvent, q workqueue.Ra Namespace: evt.Object.GetNamespace(), }}) } + +func isNil(arg any) bool { + if v := reflect.ValueOf(arg); !v.IsValid() || ((v.Kind() == reflect.Ptr || + v.Kind() == reflect.Interface || + v.Kind() == reflect.Slice || + v.Kind() == reflect.Map || + v.Kind() == reflect.Chan || + v.Kind() == reflect.Func) && v.IsNil()) { + return true + } + return false +} diff --git a/pkg/handler/enqueue_mapped.go b/pkg/handler/enqueue_mapped.go index 17401b1fdb..62d6728151 100644 --- a/pkg/handler/enqueue_mapped.go +++ b/pkg/handler/enqueue_mapped.go @@ -17,16 +17,25 @@ limitations under the License. package handler import ( + "context" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) // MapFunc is the signature required for enqueueing requests from a generic function. // This type is usually used with EnqueueRequestsFromMapFunc when registering an event handler. -type MapFunc func(client.Object) []reconcile.Request +type MapFunc = TypedMapFunc[client.Object, reconcile.Request] + +// TypedMapFunc is the signature required for enqueueing requests from a generic function. +// This type is usually used with EnqueueRequestsFromTypedMapFunc when registering an event handler. +// +// TypedMapFunc is experimental and subject to future change. +type TypedMapFunc[object any, request comparable] func(context.Context, object) []request // EnqueueRequestsFromMapFunc enqueues Requests by running a transformation function that outputs a collection // of reconcile.Requests on each Event. The reconcile.Requests may be for an arbitrary set of objects @@ -39,59 +48,106 @@ type MapFunc func(client.Object) []reconcile.Request // For UpdateEvents which contain both a new and old object, the transformation function is run on both // objects and both sets of Requests are enqueue. func EnqueueRequestsFromMapFunc(fn MapFunc) EventHandler { - return &enqueueRequestsFromMapFunc{ - toRequests: fn, + return TypedEnqueueRequestsFromMapFunc(fn) +} + +// TypedEnqueueRequestsFromMapFunc enqueues Requests by running a transformation function that outputs a collection +// of reconcile.Requests on each Event. The reconcile.Requests may be for an arbitrary set of objects +// defined by some user specified transformation of the source Event. (e.g. trigger Reconciler for a set of objects +// in response to a cluster resize event caused by adding or deleting a Node) +// +// TypedEnqueueRequestsFromMapFunc is frequently used to fan-out updates from one object to one or more other +// objects of a differing type. +// +// For TypedUpdateEvents which contain both a new and old object, the transformation function is run on both +// objects and both sets of Requests are enqueue. +// +// TypedEnqueueRequestsFromMapFunc is experimental and subject to future change. +func TypedEnqueueRequestsFromMapFunc[object any, request comparable](fn TypedMapFunc[object, request]) TypedEventHandler[object, request] { + return &enqueueRequestsFromMapFunc[object, request]{ + toRequests: fn, + objectImplementsClientObject: implementsClientObject[object](), } } -var _ EventHandler = &enqueueRequestsFromMapFunc{} +var _ EventHandler = &enqueueRequestsFromMapFunc[client.Object, reconcile.Request]{} -type enqueueRequestsFromMapFunc struct { +type enqueueRequestsFromMapFunc[object any, request comparable] struct { // Mapper transforms the argument into a slice of keys to be reconciled - toRequests MapFunc + toRequests TypedMapFunc[object, request] + objectImplementsClientObject bool } // Create implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.Object, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Create( + ctx context.Context, + evt event.TypedCreateEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + reqs := map[request]empty{} + + var lowPriority bool + if isPriorityQueue(q) && !isNil(evt.Object) { + if evt.IsInInitialList { + lowPriority = true + } + } + e.mapAndEnqueue(ctx, q, evt.Object, reqs, lowPriority) } // Update implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.ObjectOld, reqs) - e.mapAndEnqueue(q, evt.ObjectNew, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Update( + ctx context.Context, + evt event.TypedUpdateEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + var lowPriority bool + if e.objectImplementsClientObject && isPriorityQueue(q) && !isNil(evt.ObjectOld) && !isNil(evt.ObjectNew) { + lowPriority = any(evt.ObjectOld).(client.Object).GetResourceVersion() == any(evt.ObjectNew).(client.Object).GetResourceVersion() + } + reqs := map[request]empty{} + e.mapAndEnqueue(ctx, q, evt.ObjectOld, reqs, lowPriority) + e.mapAndEnqueue(ctx, q, evt.ObjectNew, reqs, lowPriority) } // Delete implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.Object, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Delete( + ctx context.Context, + evt event.TypedDeleteEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + reqs := map[request]empty{} + e.mapAndEnqueue(ctx, q, evt.Object, reqs, false) } // Generic implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.Object, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Generic( + ctx context.Context, + evt event.TypedGenericEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + reqs := map[request]empty{} + e.mapAndEnqueue(ctx, q, evt.Object, reqs, false) } -func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(q workqueue.RateLimitingInterface, object client.Object, reqs map[reconcile.Request]empty) { - for _, req := range e.toRequests(object) { +func (e *enqueueRequestsFromMapFunc[object, request]) mapAndEnqueue( + ctx context.Context, + q workqueue.TypedRateLimitingInterface[request], + o object, + reqs map[request]empty, + lowPriority bool, +) { + for _, req := range e.toRequests(ctx, o) { _, ok := reqs[req] if !ok { - q.Add(req) + if lowPriority { + q.(priorityqueue.PriorityQueue[request]).AddWithOpts(priorityqueue.AddOpts{ + Priority: ptr.To(LowPriority), + }, req) + } else { + q.Add(req) + } reqs[req] = empty{} } } } - -// EnqueueRequestsFromMapFunc can inject fields into the mapper. - -// InjectFunc implements inject.Injector. -func (e *enqueueRequestsFromMapFunc) InjectFunc(f inject.Func) error { - if f == nil { - return nil - } - return f(e.toRequests) -} diff --git a/pkg/handler/enqueue_owner.go b/pkg/handler/enqueue_owner.go index 63699893fc..e8fc8eb46e 100644 --- a/pkg/handler/enqueue_owner.go +++ b/pkg/handler/enqueue_owner.go @@ -17,6 +17,7 @@ limitations under the License. package handler import ( + "context" "fmt" "k8s.io/apimachinery/pkg/api/meta" @@ -25,15 +26,18 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) -var _ EventHandler = &EnqueueRequestForOwner{} +var _ EventHandler = &enqueueRequestForOwner[client.Object]{} -var log = logf.RuntimeLog.WithName("eventhandler").WithName("EnqueueRequestForOwner") +var log = logf.RuntimeLog.WithName("eventhandler").WithName("enqueueRequestForOwner") + +// OwnerOption modifies an EnqueueRequestForOwner EventHandler. +type OwnerOption func(e enqueueRequestForOwnerInterface) // EnqueueRequestForOwner enqueues Requests for the Owners of an object. E.g. the object that created // the object that was the source of the Event. @@ -42,13 +46,52 @@ var log = logf.RuntimeLog.WithName("eventhandler").WithName("EnqueueRequestForOw // // - a source.Kind Source with Type of Pod. // -// - a handler.EnqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and IsController set to true. -type EnqueueRequestForOwner struct { - // OwnerType is the type of the Owner object to look for in OwnerReferences. Only Group and Kind are compared. - OwnerType runtime.Object +// - a handler.enqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and OnlyControllerOwner set to true. +func EnqueueRequestForOwner(scheme *runtime.Scheme, mapper meta.RESTMapper, ownerType client.Object, opts ...OwnerOption) EventHandler { + return TypedEnqueueRequestForOwner[client.Object](scheme, mapper, ownerType, opts...) +} - // IsController if set will only look at the first OwnerReference with Controller: true. - IsController bool +// TypedEnqueueRequestForOwner enqueues Requests for the Owners of an object. E.g. the object that created +// the object that was the source of the Event. +// +// If a ReplicaSet creates Pods, users may reconcile the ReplicaSet in response to Pod Events using: +// +// - a source.Kind Source with Type of Pod. +// +// - a handler.typedEnqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and OnlyControllerOwner set to true. +// +// TypedEnqueueRequestForOwner is experimental and subject to future change. +func TypedEnqueueRequestForOwner[object client.Object](scheme *runtime.Scheme, mapper meta.RESTMapper, ownerType client.Object, opts ...OwnerOption) TypedEventHandler[object, reconcile.Request] { + e := &enqueueRequestForOwner[object]{ + ownerType: ownerType, + mapper: mapper, + } + if err := e.parseOwnerTypeGroupKind(scheme); err != nil { + panic(err) + } + for _, opt := range opts { + opt(e) + } + return WithLowPriorityWhenUnchanged(e) +} + +// OnlyControllerOwner if provided will only look at the first OwnerReference with Controller: true. +func OnlyControllerOwner() OwnerOption { + return func(e enqueueRequestForOwnerInterface) { + e.setIsController(true) + } +} + +type enqueueRequestForOwnerInterface interface { + setIsController(bool) +} + +type enqueueRequestForOwner[object client.Object] struct { + // ownerType is the type of the Owner object to look for in OwnerReferences. Only Group and Kind are compared. + ownerType runtime.Object + + // isController if set will only look at the first OwnerReference with Controller: true. + isController bool // groupKind is the cached Group and Kind from OwnerType groupKind schema.GroupKind @@ -57,8 +100,12 @@ type EnqueueRequestForOwner struct { mapper meta.RESTMapper } +func (e *enqueueRequestForOwner[object]) setIsController(isController bool) { + e.isController = isController +} + // Create implements EventHandler. -func (e *EnqueueRequestForOwner) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Create(ctx context.Context, evt event.TypedCreateEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -67,7 +114,7 @@ func (e *EnqueueRequestForOwner) Create(evt event.CreateEvent, q workqueue.RateL } // Update implements EventHandler. -func (e *EnqueueRequestForOwner) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Update(ctx context.Context, evt event.TypedUpdateEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.ObjectOld, reqs) e.getOwnerReconcileRequest(evt.ObjectNew, reqs) @@ -77,7 +124,7 @@ func (e *EnqueueRequestForOwner) Update(evt event.UpdateEvent, q workqueue.RateL } // Delete implements EventHandler. -func (e *EnqueueRequestForOwner) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Delete(ctx context.Context, evt event.TypedDeleteEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -86,7 +133,7 @@ func (e *EnqueueRequestForOwner) Delete(evt event.DeleteEvent, q workqueue.RateL } // Generic implements EventHandler. -func (e *EnqueueRequestForOwner) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Generic(ctx context.Context, evt event.TypedGenericEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -96,17 +143,17 @@ func (e *EnqueueRequestForOwner) Generic(evt event.GenericEvent, q workqueue.Rat // parseOwnerTypeGroupKind parses the OwnerType into a Group and Kind and caches the result. Returns false // if the OwnerType could not be parsed using the scheme. -func (e *EnqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error { +func (e *enqueueRequestForOwner[object]) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error { // Get the kinds of the type - kinds, _, err := scheme.ObjectKinds(e.OwnerType) + kinds, _, err := scheme.ObjectKinds(e.ownerType) if err != nil { - log.Error(err, "Could not get ObjectKinds for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType)) + log.Error(err, "Could not get ObjectKinds for OwnerType", "owner type", fmt.Sprintf("%T", e.ownerType)) return err } // Expect only 1 kind. If there is more than one kind this is probably an edge case such as ListOptions. if len(kinds) != 1 { - err := fmt.Errorf("expected exactly 1 kind for OwnerType %T, but found %s kinds", e.OwnerType, kinds) - log.Error(nil, "expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType), "kinds", kinds) + err := fmt.Errorf("expected exactly 1 kind for OwnerType %T, but found %s kinds", e.ownerType, kinds) + log.Error(nil, "expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.ownerType), "kinds", kinds) return err } // Cache the Group and Kind for the OwnerType @@ -116,10 +163,10 @@ func (e *EnqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) // getOwnerReconcileRequest looks at object and builds a map of reconcile.Request to reconcile // owners of object that match e.OwnerType. -func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, result map[reconcile.Request]empty) { +func (e *enqueueRequestForOwner[object]) getOwnerReconcileRequest(obj metav1.Object, result map[reconcile.Request]empty) { // Iterate through the OwnerReferences looking for a match on Group and Kind against what was requested // by the user - for _, ref := range e.getOwnersReferences(object) { + for _, ref := range e.getOwnersReferences(obj) { // Parse the Group out of the OwnerReference to compare it to what was parsed out of the requested OwnerType refGV, err := schema.ParseGroupVersion(ref.APIVersion) if err != nil { @@ -138,14 +185,14 @@ func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, Name: ref.Name, }} - // if owner is not namespaced then we should set the namespace to the empty + // if owner is not namespaced then we should not set the namespace mapping, err := e.mapper.RESTMapping(e.groupKind, refGV.Version) if err != nil { log.Error(err, "Could not retrieve rest mapping", "kind", e.groupKind) return } if mapping.Scope.Name() != meta.RESTScopeNameRoot { - request.Namespace = object.GetNamespace() + request.Namespace = obj.GetNamespace() } result[request] = empty{} @@ -153,37 +200,22 @@ func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, } } -// getOwnersReferences returns the OwnerReferences for an object as specified by the EnqueueRequestForOwner +// getOwnersReferences returns the OwnerReferences for an object as specified by the enqueueRequestForOwner // - if IsController is true: only take the Controller OwnerReference (if found) // - if IsController is false: take all OwnerReferences. -func (e *EnqueueRequestForOwner) getOwnersReferences(object metav1.Object) []metav1.OwnerReference { - if object == nil { +func (e *enqueueRequestForOwner[object]) getOwnersReferences(obj metav1.Object) []metav1.OwnerReference { + if obj == nil { return nil } // If not filtered as Controller only, then use all the OwnerReferences - if !e.IsController { - return object.GetOwnerReferences() + if !e.isController { + return obj.GetOwnerReferences() } // If filtered to a Controller, only take the Controller OwnerReference - if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { + if ownerRef := metav1.GetControllerOf(obj); ownerRef != nil { return []metav1.OwnerReference{*ownerRef} } // No Controller OwnerReference found return nil } - -var _ inject.Scheme = &EnqueueRequestForOwner{} - -// InjectScheme is called by the Controller to provide a singleton scheme to the EnqueueRequestForOwner. -func (e *EnqueueRequestForOwner) InjectScheme(s *runtime.Scheme) error { - return e.parseOwnerTypeGroupKind(s) -} - -var _ inject.Mapper = &EnqueueRequestForOwner{} - -// InjectMapper is called by the Controller to provide the rest mapper used by the manager. -func (e *EnqueueRequestForOwner) InjectMapper(m meta.RESTMapper) error { - e.mapper = m - return nil -} diff --git a/pkg/handler/eventhandler.go b/pkg/handler/eventhandler.go index 8652d22d72..88510d29ed 100644 --- a/pkg/handler/eventhandler.go +++ b/pkg/handler/eventhandler.go @@ -17,13 +17,21 @@ limitations under the License. package handler import ( + "context" + "reflect" + "time" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // EventHandler enqueues reconcile.Requests in response to events (e.g. Pod Create). EventHandlers map an Event // for one object to trigger Reconciles for either the same object or different objects - e.g. if there is an -// Event for object with type Foo (using source.KindSource) then reconcile one or more object(s) with type Bar. +// Event for object with type Foo (using source.Kind) then reconcile one or more object(s) with type Bar. // // Identical reconcile.Requests will be batched together through the queuing mechanism before reconcile is called. // @@ -39,66 +47,201 @@ import ( // // Unless you are implementing your own EventHandler, you can ignore the functions on the EventHandler interface. // Most users shouldn't need to implement their own EventHandler. -type EventHandler interface { - // Create is called in response to an create event - e.g. Pod Creation. - Create(event.CreateEvent, workqueue.RateLimitingInterface) +type EventHandler = TypedEventHandler[client.Object, reconcile.Request] + +// TypedEventHandler enqueues reconcile.Requests in response to events (e.g. Pod Create). TypedEventHandlers map an Event +// for one object to trigger Reconciles for either the same object or different objects - e.g. if there is an +// Event for object with type Foo (using source.Kind) then reconcile one or more object(s) with type Bar. +// +// Identical reconcile.Requests will be batched together through the queuing mechanism before reconcile is called. +// +// * Use TypedEnqueueRequestForObject to reconcile the object the event is for +// - do this for events for the type the Controller Reconciles. (e.g. Deployment for a Deployment Controller) +// +// * Use TypedEnqueueRequestForOwner to reconcile the owner of the object the event is for +// - do this for events for the types the Controller creates. (e.g. ReplicaSets created by a Deployment Controller) +// +// * Use TypedEnqueueRequestsFromMapFunc to transform an event for an object to a reconcile of an object +// of a different type - do this for events for types the Controller may be interested in, but doesn't create. +// (e.g. If Foo responds to cluster size events, map Node events to Foo objects.) +// +// Unless you are implementing your own TypedEventHandler, you can ignore the functions on the TypedEventHandler interface. +// Most users shouldn't need to implement their own TypedEventHandler. +// +// TypedEventHandler is experimental and subject to future change. +type TypedEventHandler[object any, request comparable] interface { + // Create is called in response to a create event - e.g. Pod Creation. + Create(context.Context, event.TypedCreateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Update is called in response to an update event - e.g. Pod Updated. - Update(event.UpdateEvent, workqueue.RateLimitingInterface) + Update(context.Context, event.TypedUpdateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Delete is called in response to a delete event - e.g. Pod Deleted. - Delete(event.DeleteEvent, workqueue.RateLimitingInterface) + Delete(context.Context, event.TypedDeleteEvent[object], workqueue.TypedRateLimitingInterface[request]) // Generic is called in response to an event of an unknown type or a synthetic event triggered as a cron or // external trigger request - e.g. reconcile Autoscaling, or a Webhook. - Generic(event.GenericEvent, workqueue.RateLimitingInterface) + Generic(context.Context, event.TypedGenericEvent[object], workqueue.TypedRateLimitingInterface[request]) } var _ EventHandler = Funcs{} -// Funcs implements EventHandler. -type Funcs struct { +// Funcs implements eventhandler. +type Funcs = TypedFuncs[client.Object, reconcile.Request] + +// TypedFuncs implements eventhandler. +// +// TypedFuncs is experimental and subject to future change. +type TypedFuncs[object any, request comparable] struct { // Create is called in response to an add event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - CreateFunc func(event.CreateEvent, workqueue.RateLimitingInterface) + CreateFunc func(context.Context, event.TypedCreateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Update is called in response to an update event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - UpdateFunc func(event.UpdateEvent, workqueue.RateLimitingInterface) + UpdateFunc func(context.Context, event.TypedUpdateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Delete is called in response to a delete event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - DeleteFunc func(event.DeleteEvent, workqueue.RateLimitingInterface) + DeleteFunc func(context.Context, event.TypedDeleteEvent[object], workqueue.TypedRateLimitingInterface[request]) // GenericFunc is called in response to a generic event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - GenericFunc func(event.GenericEvent, workqueue.RateLimitingInterface) + GenericFunc func(context.Context, event.TypedGenericEvent[object], workqueue.TypedRateLimitingInterface[request]) +} + +var typeForClientObject = reflect.TypeFor[client.Object]() + +func implementsClientObject[object any]() bool { + return reflect.TypeFor[object]().Implements(typeForClientObject) +} + +func isPriorityQueue[request comparable](q workqueue.TypedRateLimitingInterface[request]) bool { + _, ok := q.(priorityqueue.PriorityQueue[request]) + return ok } // Create implements EventHandler. -func (h Funcs) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Create(ctx context.Context, e event.TypedCreateEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.CreateFunc != nil { - h.CreateFunc(e, q) + if !implementsClientObject[object]() || !isPriorityQueue(q) || isNil(e.Object) { + h.CreateFunc(ctx, e, q) + return + } + + wq := workqueueWithDefaultPriority[request]{ + // We already know that we have a priority queue, that event.Object implements + // client.Object and that its not nil + PriorityQueue: q.(priorityqueue.PriorityQueue[request]), + } + if e.IsInInitialList { + wq.priority = ptr.To(LowPriority) + } + h.CreateFunc(ctx, e, wq) } } // Delete implements EventHandler. -func (h Funcs) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Delete(ctx context.Context, e event.TypedDeleteEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.DeleteFunc != nil { - h.DeleteFunc(e, q) + h.DeleteFunc(ctx, e, q) } } // Update implements EventHandler. -func (h Funcs) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Update(ctx context.Context, e event.TypedUpdateEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.UpdateFunc != nil { - h.UpdateFunc(e, q) + if !implementsClientObject[object]() || !isPriorityQueue(q) || isNil(e.ObjectOld) || isNil(e.ObjectNew) { + h.UpdateFunc(ctx, e, q) + return + } + + wq := workqueueWithDefaultPriority[request]{ + // We already know that we have a priority queue, that event.ObjectOld and ObjectNew implement + // client.Object and that they are not nil + PriorityQueue: q.(priorityqueue.PriorityQueue[request]), + } + if any(e.ObjectOld).(client.Object).GetResourceVersion() == any(e.ObjectNew).(client.Object).GetResourceVersion() { + wq.priority = ptr.To(LowPriority) + } + h.UpdateFunc(ctx, e, wq) } } // Generic implements EventHandler. -func (h Funcs) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Generic(ctx context.Context, e event.TypedGenericEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.GenericFunc != nil { - h.GenericFunc(e, q) + h.GenericFunc(ctx, e, q) + } +} + +// LowPriority is the priority set by WithLowPriorityWhenUnchanged +const LowPriority = -100 + +// WithLowPriorityWhenUnchanged reduces the priority of events stemming from the initial listwatch or from a resync if +// and only if a priorityqueue.PriorityQueue is used. If not, it does nothing. +func WithLowPriorityWhenUnchanged[object client.Object, request comparable](u TypedEventHandler[object, request]) TypedEventHandler[object, request] { + // TypedFuncs already implements this so just wrap + return TypedFuncs[object, request]{ + CreateFunc: u.Create, + UpdateFunc: u.Update, + DeleteFunc: u.Delete, + GenericFunc: u.Generic, + } +} + +type workqueueWithDefaultPriority[request comparable] struct { + priorityqueue.PriorityQueue[request] + priority *int +} + +func (w workqueueWithDefaultPriority[request]) Add(item request) { + w.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: w.priority}, item) +} + +func (w workqueueWithDefaultPriority[request]) AddAfter(item request, after time.Duration) { + w.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: w.priority, After: after}, item) +} + +func (w workqueueWithDefaultPriority[request]) AddRateLimited(item request) { + w.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: w.priority, RateLimited: true}, item) +} + +func (w workqueueWithDefaultPriority[request]) AddWithOpts(o priorityqueue.AddOpts, items ...request) { + if o.Priority == nil { + o.Priority = w.priority + } + w.PriorityQueue.AddWithOpts(o, items...) +} + +// addToQueueCreate adds the reconcile.Request to the priorityqueue in the handler +// for Create requests if and only if the workqueue being used is of type priorityqueue.PriorityQueue[reconcile.Request] +func addToQueueCreate[T client.Object, request comparable](q workqueue.TypedRateLimitingInterface[request], evt event.TypedCreateEvent[T], item request) { + priorityQueue, isPriorityQueue := q.(priorityqueue.PriorityQueue[request]) + if !isPriorityQueue { + q.Add(item) + return + } + + var priority *int + if evt.IsInInitialList { + priority = ptr.To(LowPriority) + } + priorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: priority}, item) +} + +// addToQueueUpdate adds the reconcile.Request to the priorityqueue in the handler +// for Update requests if and only if the workqueue being used is of type priorityqueue.PriorityQueue[reconcile.Request] +func addToQueueUpdate[T client.Object, request comparable](q workqueue.TypedRateLimitingInterface[request], evt event.TypedUpdateEvent[T], item request) { + priorityQueue, isPriorityQueue := q.(priorityqueue.PriorityQueue[request]) + if !isPriorityQueue { + q.Add(item) + return + } + + var priority *int + if evt.ObjectOld.GetResourceVersion() == evt.ObjectNew.GetResourceVersion() { + priority = ptr.To(LowPriority) } + priorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: priority}, item) } diff --git a/pkg/handler/eventhandler_suite_test.go b/pkg/handler/eventhandler_suite_test.go index ebcc993915..3f6b17f337 100644 --- a/pkg/handler/eventhandler_suite_test.go +++ b/pkg/handler/eventhandler_suite_test.go @@ -19,19 +19,17 @@ package handler_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestEventhandler(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Eventhandler Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Eventhandler Suite") } var testenv *envtest.Environment diff --git a/pkg/handler/eventhandler_test.go b/pkg/handler/eventhandler_test.go index 61db62e66a..2a7453f761 100644 --- a/pkg/handler/eventhandler_test.go +++ b/pkg/handler/eventhandler_test.go @@ -17,7 +17,10 @@ package handler_test import ( - . "github.com/onsi/ginkgo" + "context" + "time" + + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" @@ -26,64 +29,61 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var _ = Describe("Eventhandler", func() { - var q workqueue.RateLimitingInterface + var q workqueue.TypedRateLimitingInterface[reconcile.Request] var instance handler.EnqueueRequestForObject var pod *corev1.Pod var mapper meta.RESTMapper - t := true BeforeEach(func() { - q = controllertest.Queue{Interface: workqueue.New()} + q = &controllertest.Queue{TypedInterface: workqueue.NewTyped[reconcile.Request]()} pod = &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "biz", Name: "baz"}, } Expect(cfg).NotTo(BeNil()) - var err error - mapper, err = apiutil.NewDiscoveryRESTMapper(cfg) + httpClient, err := rest.HTTPClientFor(cfg) + Expect(err).ShouldNot(HaveOccurred()) + mapper, err = apiutil.NewDynamicRESTMapper(cfg, httpClient) Expect(err).ShouldNot(HaveOccurred()) }) Describe("EnqueueRequestForObject", func() { - It("should enqueue a Request with the Name / Namespace of the object in the CreateEvent.", func() { + It("should enqueue a Request with the Name / Namespace of the object in the CreateEvent.", func(ctx SpecContext) { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) - It("should enqueue a Request with the Name / Namespace of the object in the DeleteEvent.", func() { + It("should enqueue a Request with the Name / Namespace of the object in the DeleteEvent.", func(ctx SpecContext) { evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) It("should enqueue a Request with the Name / Namespace of one object in the UpdateEvent.", - func() { + func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = "baz2" newPod.Namespace = "biz2" @@ -92,39 +92,33 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz2", Name: "baz2"})) }) - It("should enqueue a Request with the Name / Namespace of the object in the GenericEvent.", func() { + It("should enqueue a Request with the Name / Namespace of the object in the GenericEvent.", func(ctx SpecContext) { evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) Context("for a runtime.Object without Object", func() { - It("should do nothing if the Object is missing for a CreateEvent.", func() { + It("should do nothing if the Object is missing for a CreateEvent.", func(ctx SpecContext) { evt := event.CreateEvent{ Object: nil, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) - It("should do nothing if the Object is missing for a UpdateEvent.", func() { + It("should do nothing if the Object is missing for a UpdateEvent.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = "baz2" newPod.Namespace = "biz2" @@ -133,47 +127,41 @@ var _ = Describe("Eventhandler", func() { ObjectNew: newPod, ObjectOld: nil, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz2", Name: "baz2"})) evt.ObjectNew = nil evt.ObjectOld = pod - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ = q.Get() - Expect(i).NotTo(BeNil()) - req, ok = i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ = q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) - It("should do nothing if the Object is missing for a DeleteEvent.", func() { + It("should do nothing if the Object is missing for a DeleteEvent.", func(ctx SpecContext) { evt := event.DeleteEvent{ Object: nil, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) - It("should do nothing if the Object is missing for a GenericEvent.", func() { + It("should do nothing if the Object is missing for a GenericEvent.", func(ctx SpecContext) { evt := event.GenericEvent{ Object: nil, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) }) }) Describe("EnqueueRequestsFromMapFunc", func() { - It("should enqueue a Request with the function applied to the CreateEvent.", func() { + It("should enqueue a Request with the function applied to the CreateEvent.", func(ctx SpecContext) { req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() Expect(a).To(Equal(pod)) req = []reconcile.Request{ @@ -190,7 +178,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -203,9 +191,9 @@ var _ = Describe("Eventhandler", func() { )) }) - It("should enqueue a Request with the function applied to the DeleteEvent.", func() { + It("should enqueue a Request with the function applied to the DeleteEvent.", func(ctx SpecContext) { req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() Expect(a).To(Equal(pod)) req = []reconcile.Request{ @@ -222,7 +210,7 @@ var _ = Describe("Eventhandler", func() { evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -236,12 +224,12 @@ var _ = Describe("Eventhandler", func() { }) It("should enqueue a Request with the function applied to both objects in the UpdateEvent.", - func() { + func(ctx SpecContext) { newPod := pod.DeepCopy() req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() req = []reconcile.Request{ { @@ -258,7 +246,7 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i, _ := q.Get() @@ -268,9 +256,9 @@ var _ = Describe("Eventhandler", func() { Expect(i).To(Equal(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "biz", Name: "baz-baz"}})) }) - It("should enqueue a Request with the function applied to the GenericEvent.", func() { + It("should enqueue a Request with the function applied to the GenericEvent.", func(ctx SpecContext) { req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() Expect(a).To(Equal(pod)) req = []reconcile.Request{ @@ -287,7 +275,7 @@ var _ = Describe("Eventhandler", func() { evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -302,12 +290,8 @@ var _ = Describe("Eventhandler", func() { }) Describe("EnqueueRequestForOwner", func() { - It("should enqueue a Request with the Owner of the object in the CreateEvent.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should enqueue a Request with the Owner of the object in the CreateEvent.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -319,7 +303,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -327,12 +311,8 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request with the Owner of the object in the DeleteEvent.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should enqueue a Request with the Owner of the object in the DeleteEvent.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -344,7 +324,7 @@ var _ = Describe("Eventhandler", func() { evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -352,16 +332,12 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request with the Owners of both objects in the UpdateEvent.", func() { + It("should enqueue a Request with the Owners of both objects in the UpdateEvent.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -381,7 +357,7 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -394,15 +370,11 @@ var _ = Describe("Eventhandler", func() { )) }) - It("should enqueue a Request with the one duplicate Owner of both objects in the UpdateEvent.", func() { + It("should enqueue a Request with the one duplicate Owner of both objects in the UpdateEvent.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -422,7 +394,7 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -430,13 +402,8 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request with the Owner of the object in the GenericEvent.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - + It("should enqueue a Request with the Owner of the object in the GenericEvent.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo-parent", @@ -447,7 +414,7 @@ var _ = Describe("Eventhandler", func() { evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -455,13 +422,8 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should not enqueue a Request if there are no owners matching Group and Kind.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should not enqueue a Request if there are no owners matching Group and Kind.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { // Wrong group Name: "foo1-parent", @@ -477,28 +439,24 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) It("should enqueue a Request if there are owners matching Group "+ - "and Kind with a different version.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &autoscalingv1.HorizontalPodAutoscaler{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + "and Kind with a different version.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &autoscalingv1.HorizontalPodAutoscaler{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo-parent", Kind: "HorizontalPodAutoscaler", - APIVersion: "autoscaling/v2beta1", + APIVersion: "autoscaling/v2", }, } evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -506,12 +464,8 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request for a owner that is cluster scoped", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &corev1.Node{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should enqueue a Request for a owner that is cluster scoped", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &corev1.Node{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "node-1", @@ -522,7 +476,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -531,28 +485,19 @@ var _ = Describe("Eventhandler", func() { }) - It("should not enqueue a Request if there are no owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should not enqueue a Request if there are no owners.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) Context("with the Controller field set to true", func() { It("should enqueue reconcile.Requests for only the first the Controller if there are "+ - "multiple Controller owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + "multiple Controller owners.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -563,7 +508,7 @@ var _ = Describe("Eventhandler", func() { Name: "foo2-parent", Kind: "ReplicaSet", APIVersion: "apps/v1", - Controller: &t, + Controller: ptr.To(true), }, { Name: "foo3-parent", @@ -574,7 +519,7 @@ var _ = Describe("Eventhandler", func() { Name: "foo4-parent", Kind: "ReplicaSet", APIVersion: "apps/v1", - Controller: &t, + Controller: ptr.To(true), }, { Name: "foo5-parent", @@ -585,20 +530,15 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() Expect(i).To(Equal(reconcile.Request{ NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo2-parent"}})) }) - It("should not enqueue reconcile.Requests if there are no Controller owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should not enqueue reconcile.Requests if there are no Controller owners.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -619,32 +559,23 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) - It("should not enqueue reconcile.Requests if there are no owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should not enqueue reconcile.Requests if there are no owners.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) }) Context("with the Controller field set to false", func() { - It("should enqueue a reconcile.Requests for all owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should enqueue a reconcile.Requests for all owners.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -665,7 +596,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(3)) i1, _ := q.Get() @@ -683,12 +614,8 @@ var _ = Describe("Eventhandler", func() { }) Context("with a nil object", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should do nothing.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -699,81 +626,22 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: nil, } - instance.Create(evt, q) - Expect(q.Len()).To(Equal(0)) - }) - }) - - Context("with a multiple matching kinds", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &metav1.ListOptions{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).NotTo(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - pod.OwnerReferences = []metav1.OwnerReference{ - { - Name: "foo1-parent", - Kind: "ListOptions", - APIVersion: "meta/v1", - }, - } - evt := event.CreateEvent{ - Object: pod, - } - instance.Create(evt, q) - Expect(q.Len()).To(Equal(0)) - }) - }) - Context("with an OwnerType that cannot be resolved", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &controllertest.ErrorType{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).NotTo(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - pod.OwnerReferences = []metav1.OwnerReference{ - { - Name: "foo1-parent", - Kind: "ListOptions", - APIVersion: "meta/v1", - }, - } - evt := event.CreateEvent{ - Object: pod, - } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) }) Context("with a nil OwnerType", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{} - Expect(instance.InjectScheme(scheme.Scheme)).NotTo(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - pod.OwnerReferences = []metav1.OwnerReference{ - { - Name: "foo1-parent", - Kind: "OwnerType", - APIVersion: "meta/v1", - }, - } - evt := event.CreateEvent{ - Object: pod, - } - instance.Create(evt, q) - Expect(q.Len()).To(Equal(0)) + It("should panic", func() { + Expect(func() { + handler.EnqueueRequestForOwner(nil, nil, nil) + }).To(Panic()) }) }) Context("with an invalid APIVersion in the OwnerReference", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + It("should do nothing.", func(ctx SpecContext) { + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -784,55 +652,55 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) }) }) Describe("Funcs", func() { - failingFuncs := handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + failingFuncs := handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect CreateEvent to be called.") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect DeleteEvent to be called.") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect UpdateEvent to be called.") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect GenericEvent to be called.") }, } - It("should call CreateFunc for a CreateEvent if provided.", func() { + It("should call CreateFunc for a CreateEvent if provided.", func(ctx SpecContext) { instance := failingFuncs evt := event.CreateEvent{ Object: pod, } - instance.CreateFunc = func(evt2 event.CreateEvent, q2 workqueue.RateLimitingInterface) { + instance.CreateFunc = func(ctx context.Context, evt2 event.CreateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Create(evt, q) + instance.Create(ctx, evt, q) }) - It("should NOT call CreateFunc for a CreateEvent if NOT provided.", func() { + It("should NOT call CreateFunc for a CreateEvent if NOT provided.", func(ctx SpecContext) { instance := failingFuncs instance.CreateFunc = nil evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) }) - It("should call UpdateFunc for an UpdateEvent if provided.", func() { + It("should call UpdateFunc for an UpdateEvent if provided.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" @@ -842,16 +710,16 @@ var _ = Describe("Eventhandler", func() { } instance := failingFuncs - instance.UpdateFunc = func(evt2 event.UpdateEvent, q2 workqueue.RateLimitingInterface) { + instance.UpdateFunc = func(ctx context.Context, evt2 event.UpdateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Update(evt, q) + instance.Update(ctx, evt, q) }) - It("should NOT call UpdateFunc for an UpdateEvent if NOT provided.", func() { + It("should NOT call UpdateFunc for an UpdateEvent if NOT provided.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" @@ -859,51 +727,442 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) }) - It("should call DeleteFunc for a DeleteEvent if provided.", func() { + It("should call DeleteFunc for a DeleteEvent if provided.", func(ctx SpecContext) { instance := failingFuncs evt := event.DeleteEvent{ Object: pod, } - instance.DeleteFunc = func(evt2 event.DeleteEvent, q2 workqueue.RateLimitingInterface) { + instance.DeleteFunc = func(ctx context.Context, evt2 event.DeleteEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) }) - It("should NOT call DeleteFunc for a DeleteEvent if NOT provided.", func() { + It("should NOT call DeleteFunc for a DeleteEvent if NOT provided.", func(ctx SpecContext) { instance := failingFuncs instance.DeleteFunc = nil evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) }) - It("should call GenericFunc for a GenericEvent if provided.", func() { + It("should call GenericFunc for a GenericEvent if provided.", func(ctx SpecContext) { instance := failingFuncs evt := event.GenericEvent{ Object: pod, } - instance.GenericFunc = func(evt2 event.GenericEvent, q2 workqueue.RateLimitingInterface) { + instance.GenericFunc = func(ctx context.Context, evt2 event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) }) - It("should NOT call GenericFunc for a GenericEvent if NOT provided.", func() { + It("should NOT call GenericFunc for a GenericEvent if NOT provided.", func(ctx SpecContext) { instance := failingFuncs instance.GenericFunc = nil evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) }) }) + + Describe("WithLowPriorityWhenUnchanged", func() { + handlerPriorityTests := []struct { + name string + handler func() handler.EventHandler + after time.Duration + ratelimited bool + overridePriority int + }{ + { + name: "WithLowPriorityWhenUnchanged wrapper", + handler: func() handler.EventHandler { return handler.WithLowPriorityWhenUnchanged(customHandler{}) }, + }, + { + name: "EnqueueRequestForObject", + handler: func() handler.EventHandler { return &handler.EnqueueRequestForObject{} }, + }, + { + name: "EnqueueRequestForOwner", + handler: func() handler.EventHandler { + return handler.EnqueueRequestForOwner( + scheme.Scheme, + mapper, + &corev1.Pod{}, + ) + }, + }, + { + name: "TypedEnqueueRequestForOwner", + handler: func() handler.EventHandler { + return handler.TypedEnqueueRequestForOwner[client.Object]( + scheme.Scheme, + mapper, + &corev1.Pod{}, + ) + }, + }, + { + name: "Funcs", + handler: func() handler.EventHandler { + return handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + } + }, + }, + { + name: "EnqueueRequestsFromMapFunc", + handler: func() handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + }}} + }) + }, + }, + { + name: "WithLowPriorityWhenUnchanged - Add", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + }) + }, + }, + { + name: "WithLowPriorityWhenUnchanged - AddAfter", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddAfter(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}, time.Second) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddAfter(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}, time.Second) + }, + }) + }, + after: time.Second, + }, + { + name: "WithLowPriorityWhenUnchanged - AddRateLimited", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddRateLimited(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddRateLimited(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + }) + }, + ratelimited: true, + }, + { + name: "WithLowPriorityWhenUnchanged - AddWithOpts priority is retained", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if pq, isPQ := wq.(priorityqueue.PriorityQueue[reconcile.Request]); isPQ { + pq.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(100)}, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + return + } + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if pq, isPQ := wq.(priorityqueue.PriorityQueue[reconcile.Request]); isPQ { + pq.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(100)}, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + return + } + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + }) + }, + overridePriority: 100, + }, + } + for _, test := range handlerPriorityTests { + When("handler is "+test.name, func() { + It("should lower the priority of a create request for an object that was part of the initial list", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Create(ctx, event.CreateEvent{ + Object: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + CreationTimestamp: metav1.Now(), + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + IsInInitialList: true, + }, wq) + + expected := handler.LowPriority + if test.overridePriority != 0 { + expected = test.overridePriority + } + + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{ + Priority: ptr.To(expected), + After: test.after, + RateLimited: test.ratelimited, + })) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should not lower the priority of a create request for an object that was not part of the initial list", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Create(ctx, event.CreateEvent{ + Object: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + CreationTimestamp: metav1.Now(), + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + IsInInitialList: false, + }, wq) + + var expectedPriority *int + if test.overridePriority != 0 { + expectedPriority = &test.overridePriority + } + + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{After: test.after, RateLimited: test.ratelimited, Priority: expectedPriority})) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should lower the priority of an update request with unchanged RV", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Update(ctx, event.UpdateEvent{ + ObjectOld: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + ObjectNew: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, wq) + + expectedPriority := handler.LowPriority + if test.overridePriority != 0 { + expectedPriority = test.overridePriority + } + + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{After: test.after, RateLimited: test.ratelimited, Priority: ptr.To(expectedPriority)})) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should not lower the priority of an update request with changed RV", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Update(ctx, event.UpdateEvent{ + ObjectOld: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + ObjectNew: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + ResourceVersion: "1", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, wq) + + var expectedPriority *int + if test.overridePriority != 0 { + expectedPriority = &test.overridePriority + } + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{After: test.after, RateLimited: test.ratelimited, Priority: expectedPriority})) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should have no effect on create if the workqueue is not a priorityqueue", func(ctx SpecContext) { + test.handler().Create(ctx, event.CreateEvent{ + Object: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, q) + + Expect(q.Len()).To(Equal(1)) + item, _ := q.Get() + Expect(item).To(Equal(reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-pod"}})) + }) + + It("should have no effect on Update if the workqueue is not a priorityqueue", func(ctx SpecContext) { + test.handler().Update(ctx, event.UpdateEvent{ + ObjectOld: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + ObjectNew: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, q) + + Expect(q.Len()).To(Equal(1)) + item, _ := q.Get() + Expect(item).To(Equal(reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-pod"}})) + }) + }) + } + }) }) + +type fakePriorityQueue struct { + workqueue.TypedRateLimitingInterface[reconcile.Request] + addWithOpts func(o priorityqueue.AddOpts, items ...reconcile.Request) +} + +func (f *fakePriorityQueue) Add(item reconcile.Request) { + f.AddWithOpts(priorityqueue.AddOpts{}, item) +} + +func (f *fakePriorityQueue) AddWithOpts(o priorityqueue.AddOpts, items ...reconcile.Request) { + f.addWithOpts(o, items...) +} +func (f *fakePriorityQueue) GetWithPriority() (item reconcile.Request, priority int, shutdown bool) { + panic("GetWithPriority is not expected to be called") +} + +// customHandler re-implements the basic enqueueRequestForObject logic +// to be able to test the WithLowPriorityWhenUnchanged wrapper +type customHandler struct{} + +func (ch customHandler) Create(ctx context.Context, evt event.CreateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.Object.GetNamespace(), + Name: evt.Object.GetName(), + }}) +} +func (ch customHandler) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.ObjectNew.GetNamespace(), + Name: evt.ObjectNew.GetName(), + }}) +} +func (ch customHandler) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.Object.GetNamespace(), + Name: evt.Object.GetName(), + }}) +} +func (ch customHandler) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.Object.GetNamespace(), + Name: evt.Object.GetName(), + }}) +} diff --git a/pkg/handler/example_test.go b/pkg/handler/example_test.go index dbfab46157..ad87e4be63 100644 --- a/pkg/handler/example_test.go +++ b/pkg/handler/example_test.go @@ -17,27 +17,31 @@ limitations under the License. package handler_test import ( + "context" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) -var c controller.Controller +var ( + mgr manager.Manager + c controller.Controller +) // This example watches Pods and enqueues Requests with the Name and Namespace of the Pod from // the Event (i.e. change caused by a Create, Update, Delete). func ExampleEnqueueRequestForObject() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &corev1.Pod{}}, - &handler.EnqueueRequestForObject{}, + source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), ) if err != nil { // handle it @@ -49,11 +53,10 @@ func ExampleEnqueueRequestForObject() { func ExampleEnqueueRequestForOwner() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &appsv1.ReplicaSet{}}, - &handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.Deployment{}, - IsController: true, - }, + source.Kind(mgr.GetCache(), + &appsv1.ReplicaSet{}, + handler.TypedEnqueueRequestForOwner[*appsv1.ReplicaSet](mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), + ), ) if err != nil { // handle it @@ -65,19 +68,20 @@ func ExampleEnqueueRequestForOwner() { func ExampleEnqueueRequestsFromMapFunc() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &appsv1.Deployment{}}, - handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: a.GetName() + "-1", - Namespace: a.GetNamespace(), - }}, - {NamespacedName: types.NamespacedName{ - Name: a.GetName() + "-2", - Namespace: a.GetNamespace(), - }}, - } - }), + source.Kind(mgr.GetCache(), &appsv1.Deployment{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, a *appsv1.Deployment) []reconcile.Request { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: a.Name + "-1", + Namespace: a.Namespace, + }}, + {NamespacedName: types.NamespacedName{ + Name: a.Name + "-2", + Namespace: a.Namespace, + }}, + } + }), + ), ) if err != nil { // handle it @@ -88,33 +92,34 @@ func ExampleEnqueueRequestsFromMapFunc() { func ExampleFuncs() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &corev1.Pod{}}, - handler.Funcs{ - CreateFunc: func(e event.CreateEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.Object.GetName(), - Namespace: e.Object.GetNamespace(), - }}) - }, - UpdateFunc: func(e event.UpdateEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.ObjectNew.GetName(), - Namespace: e.ObjectNew.GetNamespace(), - }}) - }, - DeleteFunc: func(e event.DeleteEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.Object.GetName(), - Namespace: e.Object.GetNamespace(), - }}) - }, - GenericFunc: func(e event.GenericEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.Object.GetName(), - Namespace: e.Object.GetNamespace(), - }}) + source.Kind(mgr.GetCache(), &corev1.Pod{}, + handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.Object.Name, + Namespace: e.Object.Namespace, + }}) + }, + UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.ObjectNew.Name, + Namespace: e.ObjectNew.Namespace, + }}) + }, + DeleteFunc: func(ctx context.Context, e event.TypedDeleteEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.Object.Name, + Namespace: e.Object.Namespace, + }}) + }, + GenericFunc: func(ctx context.Context, e event.TypedGenericEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.Object.Name, + Namespace: e.Object.Namespace, + }}) + }, }, - }, + ), ) if err != nil { // handle it diff --git a/pkg/healthz/healthz.go b/pkg/healthz/healthz.go index bd1cc151af..cfb5dc8d02 100644 --- a/pkg/healthz/healthz.go +++ b/pkg/healthz/healthz.go @@ -70,7 +70,7 @@ func (h *Handler) serveAggregated(resp http.ResponseWriter, req *http.Request) { parts = append(parts, checkStatus{name: "ping", healthy: true}) } - for _, c := range excluded.List() { + for _, c := range excluded.UnsortedList() { log.V(1).Info("cannot exclude health check, no matches for it", "checker", c) } @@ -88,7 +88,7 @@ func (h *Handler) serveAggregated(resp http.ResponseWriter, req *http.Request) { // any checks that the user requested to have excluded, but weren't actually // known checks. writeStatusAsText is always verbose on failure, and can be // forced to be verbose on success using the given argument. -func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, unknownExcludes sets.String, failed, forceVerbose bool) { +func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, unknownExcludes sets.Set[string], failed, forceVerbose bool) { resp.Header().Set("Content-Type", "text/plain; charset=utf-8") resp.Header().Set("X-Content-Type-Options", "nosniff") @@ -121,7 +121,7 @@ func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, unknownE } if unknownExcludes.Len() > 0 { - fmt.Fprintf(resp, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(unknownExcludes.List()...)) + fmt.Fprintf(resp, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(unknownExcludes.UnsortedList()...)) } if failed { @@ -187,12 +187,12 @@ type Checker func(req *http.Request) error var Ping Checker = func(_ *http.Request) error { return nil } // getExcludedChecks extracts the health check names to be excluded from the query param. -func getExcludedChecks(r *http.Request) sets.String { +func getExcludedChecks(r *http.Request) sets.Set[string] { checks, found := r.URL.Query()["exclude"] if found { - return sets.NewString(checks...) + return sets.New[string](checks...) } - return sets.NewString() + return sets.New[string]() } // formatQuoted returns a formatted string of the health check names, diff --git a/pkg/healthz/healthz_suite_test.go b/pkg/healthz/healthz_suite_test.go index b51fcb3605..8e16a58aa0 100644 --- a/pkg/healthz/healthz_suite_test.go +++ b/pkg/healthz/healthz_suite_test.go @@ -19,17 +19,15 @@ package healthz_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestHealthz(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Healthz Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Healthz Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/healthz/healthz_test.go b/pkg/healthz/healthz_test.go index e0413103f7..639a7575f3 100644 --- a/pkg/healthz/healthz_test.go +++ b/pkg/healthz/healthz_test.go @@ -21,7 +21,7 @@ import ( "net/http" "net/http/httptest" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/healthz" ) diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go index 87431a438f..7dd06957eb 100644 --- a/pkg/internal/controller/controller.go +++ b/pkg/internal/controller/controller.go @@ -21,24 +21,75 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" "github.com/go-logr/logr" + "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/handler" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics" logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" ) -var _ inject.Injector = &Controller{} +// Options are the arguments for creating a new Controller. +type Options[request comparable] struct { + // Reconciler is a function that can be called at any time with the Name / Namespace of an object and + // ensures that the state of the system matches the state specified in the object. + // Defaults to the DefaultReconcileFunc. + Do reconcile.TypedReconciler[request] + + // RateLimiter is used to limit how frequently requests may be queued into the work queue. + RateLimiter workqueue.TypedRateLimiter[request] + + // NewQueue constructs the queue for this controller once the controller is ready to start. + // This is a func because the standard Kubernetes work queues start themselves immediately, which + // leads to goroutine leaks if something calls controller.New repeatedly. + NewQueue func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. + MaxConcurrentReconciles int + + // CacheSyncTimeout refers to the time limit set on waiting for cache to sync + // Defaults to 2 minutes if not set. + CacheSyncTimeout time.Duration + + // Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required. + Name string + + // LogConstructor is used to construct a logger to then log messages to users during reconciliation, + // or for example when a watch is started. + // Note: LogConstructor has to be able to handle nil requests as we are also using it + // outside the context of a reconciliation. + LogConstructor func(request *request) logr.Logger + + // RecoverPanic indicates whether the panic caused by reconcile should be recovered. + // Defaults to true. + RecoverPanic *bool + + // LeaderElected indicates whether the controller is leader elected or always running. + LeaderElected *bool + + // EnableWarmup specifies whether the controller should start its sources + // when the manager is not the leader. + // Defaults to false, which means that the controller will wait for leader election to start + // before starting sources. + EnableWarmup *bool + + // ReconciliationTimeout is used as the timeout passed to the context of each Reconcile call. + // By default, there is no timeout. + ReconciliationTimeout time.Duration +} // Controller implements controller.Controller. -type Controller struct { +type Controller[request comparable] struct { // Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required. Name string @@ -48,20 +99,19 @@ type Controller struct { // Reconciler is a function that can be called at any time with the Name / Namespace of an object and // ensures that the state of the system matches the state specified in the object. // Defaults to the DefaultReconcileFunc. - Do reconcile.Reconciler + Do reconcile.TypedReconciler[request] + + // RateLimiter is used to limit how frequently requests may be queued into the work queue. + RateLimiter workqueue.TypedRateLimiter[request] - // MakeQueue constructs the queue for this controller once the controller is ready to start. - // This exists because the standard Kubernetes workqueues start themselves immediately, which + // NewQueue constructs the queue for this controller once the controller is ready to start. + // This is a func because the standard Kubernetes work queues start themselves immediately, which // leads to goroutine leaks if something calls controller.New repeatedly. - MakeQueue func() workqueue.RateLimitingInterface + NewQueue func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] // Queue is an listeningQueue that listens for events from Informers and adds object keys to // the Queue for processing - Queue workqueue.RateLimitingInterface - - // SetFields is used to inject dependencies into other objects such as Sources, EventHandlers and Predicates - // Deprecated: the caller should handle injected fields itself. - SetFields func(i interface{}) error + Queue priorityqueue.PriorityQueue[request] // mu is used to synchronize Controller setup mu sync.Mutex @@ -81,71 +131,132 @@ type Controller struct { CacheSyncTimeout time.Duration // startWatches maintains a list of sources, handlers, and predicates to start when the controller is started. - startWatches []watchDescription + startWatches []source.TypedSource[request] + + // startedEventSourcesAndQueue is used to track if the event sources have been started. + // It ensures that we append sources to c.startWatches only until we call Start() / Warmup() + // It is true if startEventSourcesAndQueueLocked has been called at least once. + startedEventSourcesAndQueue bool - // Log is used to log messages to users during reconciliation, or for example when a watch is started. - Log logr.Logger + // didStartEventSourcesOnce is used to ensure that the event sources are only started once. + didStartEventSourcesOnce sync.Once + + // LogConstructor is used to construct a logger to then log messages to users during reconciliation, + // or for example when a watch is started. + // Note: LogConstructor has to be able to handle nil requests as we are also using it + // outside the context of a reconciliation. + LogConstructor func(request *request) logr.Logger // RecoverPanic indicates whether the panic caused by reconcile should be recovered. - RecoverPanic bool + // Defaults to true. + RecoverPanic *bool + + // LeaderElected indicates whether the controller is leader elected or always running. + LeaderElected *bool + + // EnableWarmup specifies whether the controller should start its sources when the manager is not + // the leader. This is useful for cases where sources take a long time to start, as it allows + // for the controller to warm up its caches even before it is elected as the leader. This + // improves leadership failover time, as the caches will be prepopulated before the controller + // transitions to be leader. + // + // Setting EnableWarmup to true and NeedLeaderElection to true means the controller will start its + // sources without waiting to become leader. + // Setting EnableWarmup to true and NeedLeaderElection to false is a no-op as controllers without + // leader election do not wait on leader election to start their sources. + // Defaults to false. + EnableWarmup *bool + + ReconciliationTimeout time.Duration } -// watchDescription contains all the information necessary to start a watch. -type watchDescription struct { - src source.Source - handler handler.EventHandler - predicates []predicate.Predicate +// New returns a new Controller configured with the given options. +func New[request comparable](options Options[request]) *Controller[request] { + return &Controller[request]{ + Do: options.Do, + RateLimiter: options.RateLimiter, + NewQueue: options.NewQueue, + MaxConcurrentReconciles: options.MaxConcurrentReconciles, + CacheSyncTimeout: options.CacheSyncTimeout, + Name: options.Name, + LogConstructor: options.LogConstructor, + RecoverPanic: options.RecoverPanic, + LeaderElected: options.LeaderElected, + EnableWarmup: options.EnableWarmup, + ReconciliationTimeout: options.ReconciliationTimeout, + } } // Reconcile implements reconcile.Reconciler. -func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { - if c.RecoverPanic { - defer func() { - if r := recover(); r != nil { +func (c *Controller[request]) Reconcile(ctx context.Context, req request) (_ reconcile.Result, err error) { + defer func() { + if r := recover(); r != nil { + ctrlmetrics.ReconcilePanics.WithLabelValues(c.Name).Inc() + + if c.RecoverPanic == nil || *c.RecoverPanic { for _, fn := range utilruntime.PanicHandlers { - fn(r) + fn(ctx, r) } err = fmt.Errorf("panic: %v [recovered]", r) + return } - }() + + log := logf.FromContext(ctx) + log.Info(fmt.Sprintf("Observed a panic in reconciler: %v", r)) + panic(r) + } + }() + + if c.ReconciliationTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, c.ReconciliationTimeout) + defer cancel() } - log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace) - ctx = logf.IntoContext(ctx, log) + return c.Do.Reconcile(ctx, req) } // Watch implements controller.Controller. -func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prct ...predicate.Predicate) error { +func (c *Controller[request]) Watch(src source.TypedSource[request]) error { c.mu.Lock() defer c.mu.Unlock() - // Inject Cache into arguments - if err := c.SetFields(src); err != nil { - return err - } - if err := c.SetFields(evthdler); err != nil { - return err + // Sources weren't started yet, store the watches locally and return. + // These sources are going to be held until either Warmup() or Start(...) is called. + if !c.startedEventSourcesAndQueue { + c.startWatches = append(c.startWatches, src) + return nil } - for _, pr := range prct { - if err := c.SetFields(pr); err != nil { - return err - } + + c.LogConstructor(nil).Info("Starting EventSource", "source", src) + return src.Start(c.ctx, c.Queue) +} + +// NeedLeaderElection implements the manager.LeaderElectionRunnable interface. +func (c *Controller[request]) NeedLeaderElection() bool { + if c.LeaderElected == nil { + return true } + return *c.LeaderElected +} - // Controller hasn't started yet, store the watches locally and return. - // - // These watches are going to be held on the controller struct until the manager or user calls Start(...). - if !c.Started { - c.startWatches = append(c.startWatches, watchDescription{src: src, handler: evthdler, predicates: prct}) +// Warmup implements the manager.WarmupRunnable interface. +func (c *Controller[request]) Warmup(ctx context.Context) error { + if c.EnableWarmup == nil || !*c.EnableWarmup { return nil } - c.Log.Info("Starting EventSource", "source", src) - return src.Start(c.ctx, evthdler, c.Queue, prct...) + c.mu.Lock() + defer c.mu.Unlock() + + // Set the ctx so later calls to watch use this internal context + c.ctx = ctx + + return c.startEventSourcesAndQueueLocked(ctx) } // Start implements controller.Controller. -func (c *Controller) Start(ctx context.Context) error { +func (c *Controller[request]) Start(ctx context.Context) error { // use an IIFE to get proper lock handling // but lock outside to get proper handling of the queue shutdown c.mu.Lock() @@ -158,66 +269,24 @@ func (c *Controller) Start(ctx context.Context) error { // Set the internal context. c.ctx = ctx - c.Queue = c.MakeQueue() - go func() { - <-ctx.Done() - c.Queue.ShutDown() - }() - wg := &sync.WaitGroup{} err := func() error { defer c.mu.Unlock() // TODO(pwittrock): Reconsider HandleCrash - defer utilruntime.HandleCrash() + defer utilruntime.HandleCrashWithLogger(c.LogConstructor(nil)) // NB(directxman12): launch the sources *before* trying to wait for the - // caches to sync so that they have a chance to register their intendeded + // caches to sync so that they have a chance to register their intended // caches. - for _, watch := range c.startWatches { - c.Log.Info("Starting EventSource", "source", watch.src) - - if err := watch.src.Start(ctx, watch.handler, c.Queue, watch.predicates...); err != nil { - return err - } - } - - // Start the SharedIndexInformer factories to begin populating the SharedIndexInformer caches - c.Log.Info("Starting Controller") - - for _, watch := range c.startWatches { - syncingSource, ok := watch.src.(source.SyncingSource) - if !ok { - continue - } - - if err := func() error { - // use a context with timeout for launching sources and syncing caches. - sourceStartCtx, cancel := context.WithTimeout(ctx, c.CacheSyncTimeout) - defer cancel() - - // WaitForSync waits for a definitive timeout, and returns if there - // is an error or a timeout - if err := syncingSource.WaitForSync(sourceStartCtx); err != nil { - err := fmt.Errorf("failed to wait for %s caches to sync: %w", c.Name, err) - c.Log.Error(err, "Could not wait for Cache to sync") - return err - } - - return nil - }(); err != nil { - return err - } + if err := c.startEventSourcesAndQueueLocked(ctx); err != nil { + return err } - // All the watches have been started, we can reset the local slice. - // - // We should never hold watches more than necessary, each watch source can hold a backing cache, - // which won't be garbage collected if we hold a reference to it. - c.startWatches = nil + c.LogConstructor(nil).Info("Starting Controller") // Launch workers to process resources - c.Log.Info("Starting workers", "worker count", c.MaxConcurrentReconciles) + c.LogConstructor(nil).Info("Starting workers", "worker count", c.MaxConcurrentReconciles) wg.Add(c.MaxConcurrentReconciles) for i := 0; i < c.MaxConcurrentReconciles; i++ { go func() { @@ -237,16 +306,102 @@ func (c *Controller) Start(ctx context.Context) error { } <-ctx.Done() - c.Log.Info("Shutdown signal received, waiting for all workers to finish") + c.LogConstructor(nil).Info("Shutdown signal received, waiting for all workers to finish") wg.Wait() - c.Log.Info("All workers finished") + c.LogConstructor(nil).Info("All workers finished") return nil } +// startEventSourcesAndQueueLocked launches all the sources registered with this controller and waits +// for them to sync. It returns an error if any of the sources fail to start or sync. +func (c *Controller[request]) startEventSourcesAndQueueLocked(ctx context.Context) error { + var retErr error + + c.didStartEventSourcesOnce.Do(func() { + queue := c.NewQueue(c.Name, c.RateLimiter) + if priorityQueue, isPriorityQueue := queue.(priorityqueue.PriorityQueue[request]); isPriorityQueue { + c.Queue = priorityQueue + } else { + c.Queue = &priorityQueueWrapper[request]{TypedRateLimitingInterface: queue} + } + go func() { + <-ctx.Done() + c.Queue.ShutDown() + }() + + errGroup := &errgroup.Group{} + for _, watch := range c.startWatches { + log := c.LogConstructor(nil) + _, ok := watch.(interface { + String() string + }) + if !ok { + log = log.WithValues("source", fmt.Sprintf("%T", watch)) + } else { + log = log.WithValues("source", fmt.Sprintf("%s", watch)) + } + didStartSyncingSource := &atomic.Bool{} + errGroup.Go(func() error { + // Use a timeout for starting and syncing the source to avoid silently + // blocking startup indefinitely if it doesn't come up. + sourceStartCtx, cancel := context.WithTimeout(ctx, c.CacheSyncTimeout) + defer cancel() + + sourceStartErrChan := make(chan error, 1) // Buffer chan to not leak goroutine if we time out + go func() { + defer close(sourceStartErrChan) + log.Info("Starting EventSource") + + if err := watch.Start(ctx, c.Queue); err != nil { + sourceStartErrChan <- err + return + } + syncingSource, ok := watch.(source.TypedSyncingSource[request]) + if !ok { + return + } + didStartSyncingSource.Store(true) + if err := syncingSource.WaitForSync(sourceStartCtx); err != nil { + err := fmt.Errorf("failed to wait for %s caches to sync %v: %w", c.Name, syncingSource, err) + log.Error(err, "Could not wait for Cache to sync") + sourceStartErrChan <- err + } + }() + + select { + case err := <-sourceStartErrChan: + return err + case <-sourceStartCtx.Done(): + if didStartSyncingSource.Load() { // We are racing with WaitForSync, wait for it to let it tell us what happened + return <-sourceStartErrChan + } + if ctx.Err() != nil { // Don't return an error if the root context got cancelled + return nil + } + return fmt.Errorf("timed out waiting for source %s to Start. Please ensure that its Start() method is non-blocking", watch) + } + }) + } + retErr = errGroup.Wait() + + // All the watches have been started, we can reset the local slice. + // + // We should never hold watches more than necessary, each watch source can hold a backing cache, + // which won't be garbage collected if we hold a reference to it. + c.startWatches = nil + + // Mark event sources as started after resetting the startWatches slice so that watches from + // a new Watch() call are immediately started. + c.startedEventSourcesAndQueue = true + }) + + return retErr +} + // processNextWorkItem will read a single work item off the workqueue and // attempt to process it, by calling the reconcileHandler. -func (c *Controller) processNextWorkItem(ctx context.Context) bool { - obj, shutdown := c.Queue.Get() +func (c *Controller[request]) processNextWorkItem(ctx context.Context) bool { + obj, priority, shutdown := c.Queue.GetWithPriority() if shutdown { // Stop working return false @@ -263,7 +418,7 @@ func (c *Controller) processNextWorkItem(ctx context.Context) bool { ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(1) defer ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(-1) - c.reconcileHandler(ctx, obj) + c.reconcileHandler(ctx, obj, priority) return true } @@ -274,78 +429,120 @@ const ( labelSuccess = "success" ) -func (c *Controller) initMetrics() { - ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Set(0) - ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Add(0) +func (c *Controller[request]) initMetrics() { ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Add(0) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Add(0) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Add(0) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Add(0) + ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Add(0) + ctrlmetrics.TerminalReconcileErrors.WithLabelValues(c.Name).Add(0) + ctrlmetrics.ReconcilePanics.WithLabelValues(c.Name).Add(0) ctrlmetrics.WorkerCount.WithLabelValues(c.Name).Set(float64(c.MaxConcurrentReconciles)) + ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Set(0) } -func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) { +func (c *Controller[request]) reconcileHandler(ctx context.Context, req request, priority int) { // Update metrics after processing each item reconcileStartTS := time.Now() defer func() { c.updateMetrics(time.Since(reconcileStartTS)) }() - // Make sure that the the object is a valid request. - req, ok := obj.(reconcile.Request) - if !ok { - // As the item in the workqueue is actually invalid, we call - // Forget here else we'd go into a loop of attempting to - // process a work item that is invalid. - c.Queue.Forget(obj) - c.Log.Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj) - // Return true, don't take a break - return - } + log := c.LogConstructor(&req) + reconcileID := uuid.NewUUID() - log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace) + log = log.WithValues("reconcileID", reconcileID) ctx = logf.IntoContext(ctx, log) + ctx = addReconcileID(ctx, reconcileID) // RunInformersAndControllers the syncHandler, passing it the Namespace/Name string of the // resource to be synced. + log.V(5).Info("Reconciling") result, err := c.Reconcile(ctx, req) + if result.Priority != nil { + priority = *result.Priority + } switch { case err != nil: - c.Queue.AddRateLimited(req) + if errors.Is(err, reconcile.TerminalError(nil)) { + ctrlmetrics.TerminalReconcileErrors.WithLabelValues(c.Name).Inc() + } else { + c.Queue.AddWithOpts(priorityqueue.AddOpts{RateLimited: true, Priority: ptr.To(priority)}, req) + } ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc() ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Inc() + if result.RequeueAfter > 0 || result.Requeue { //nolint: staticcheck // We have to handle Requeue until it is removed + log.Info("Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler") + } log.Error(err, "Reconciler error") case result.RequeueAfter > 0: + log.V(5).Info(fmt.Sprintf("Reconcile done, requeueing after %s", result.RequeueAfter)) // The result.RequeueAfter request will be lost, if it is returned // along with a non-nil error. But this is intended as // We need to drive to stable reconcile loops before queuing due // to result.RequestAfter - c.Queue.Forget(obj) - c.Queue.AddAfter(req, result.RequeueAfter) + c.Queue.Forget(req) + c.Queue.AddWithOpts(priorityqueue.AddOpts{After: result.RequeueAfter, Priority: ptr.To(priority)}, req) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Inc() - case result.Requeue: - c.Queue.AddRateLimited(req) + case result.Requeue: //nolint: staticcheck // We have to handle it until it is removed + log.V(5).Info("Reconcile done, requeueing") + c.Queue.AddWithOpts(priorityqueue.AddOpts{RateLimited: true, Priority: ptr.To(priority)}, req) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Inc() default: + log.V(5).Info("Reconcile successful") // Finally, if no error occurs we Forget this item so it does not // get queued again until another change happens. - c.Queue.Forget(obj) + c.Queue.Forget(req) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Inc() } } // GetLogger returns this controller's logger. -func (c *Controller) GetLogger() logr.Logger { - return c.Log -} - -// InjectFunc implement SetFields.Injector. -func (c *Controller) InjectFunc(f inject.Func) error { - c.SetFields = f - return nil +func (c *Controller[request]) GetLogger() logr.Logger { + return c.LogConstructor(nil) } // updateMetrics updates prometheus metrics within the controller. -func (c *Controller) updateMetrics(reconcileTime time.Duration) { +func (c *Controller[request]) updateMetrics(reconcileTime time.Duration) { ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds()) } + +// ReconcileIDFromContext gets the reconcileID from the current context. +func ReconcileIDFromContext(ctx context.Context) types.UID { + r, ok := ctx.Value(reconcileIDKey{}).(types.UID) + if !ok { + return "" + } + + return r +} + +// reconcileIDKey is a context.Context Value key. Its associated value should +// be a types.UID. +type reconcileIDKey struct{} + +func addReconcileID(ctx context.Context, reconcileID types.UID) context.Context { + return context.WithValue(ctx, reconcileIDKey{}, reconcileID) +} + +type priorityQueueWrapper[request comparable] struct { + workqueue.TypedRateLimitingInterface[request] +} + +func (p *priorityQueueWrapper[request]) AddWithOpts(opts priorityqueue.AddOpts, items ...request) { + for _, item := range items { + switch { + case opts.RateLimited: + p.TypedRateLimitingInterface.AddRateLimited(item) + case opts.After > 0: + p.TypedRateLimitingInterface.AddAfter(item, opts.After) + default: + p.TypedRateLimitingInterface.Add(item) + } + } +} + +func (p *priorityQueueWrapper[request]) GetWithPriority() (request, int, bool) { + item, shutdown := p.TypedRateLimitingInterface.Get() + return item, 0, shutdown +} diff --git a/pkg/internal/controller/controller_suite_test.go b/pkg/internal/controller/controller_suite_test.go index 6091dd746c..3143d3dd74 100644 --- a/pkg/internal/controller/controller_suite_test.go +++ b/pkg/internal/controller/controller_suite_test.go @@ -19,20 +19,18 @@ package controller import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Controller internal Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Controller internal Suite") } var testenv *envtest.Environment @@ -50,7 +48,7 @@ var _ = BeforeSuite(func() { clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/internal/controller/controller_test.go b/pkg/internal/controller/controller_test.go index 9f23fa2abc..6d62b80e22 100644 --- a/pkg/internal/controller/controller_test.go +++ b/pkg/internal/controller/controller_test.go @@ -21,36 +21,48 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" - . "github.com/onsi/ginkgo" + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" + "go.uber.org/goleak" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics" "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/leaderelection" + fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" ) +type TestRequest struct { + Key string +} + +const testControllerName = "testcontroller" + var _ = Describe("controller", func() { var fakeReconcile *fakeReconciler - var ctrl *Controller + var ctrl *Controller[reconcile.Request] var queue *controllertest.Queue - var informers *informertest.FakeInformers var reconciled chan reconcile.Request var request = reconcile.Request{ NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}, @@ -63,23 +75,22 @@ var _ = Describe("controller", func() { results: make(chan fakeReconcileResultPair, 10 /* chosen by the completely scientific approach of guessing */), } queue = &controllertest.Queue{ - Interface: workqueue.New(), + TypedInterface: workqueue.NewTyped[reconcile.Request](), } - informers = &informertest.FakeInformers{} - ctrl = &Controller{ + ctrl = New[reconcile.Request](Options[reconcile.Request]{ MaxConcurrentReconciles: 1, Do: fakeReconcile, - MakeQueue: func() workqueue.RateLimitingInterface { return queue }, - Log: log.RuntimeLog.WithName("controller").WithName("test"), - } - Expect(ctrl.InjectFunc(func(interface{}) error { return nil })).To(Succeed()) + NewQueue: func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return queue + }, + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return log.RuntimeLog.WithName("controller").WithName("test") + }, + }) }) Describe("Reconciler", func() { - It("should call the Reconciler function", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - + It("should call the Reconciler function", func(ctx SpecContext) { ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { return reconcile.Result{Requeue: true}, nil }) @@ -89,13 +100,11 @@ var _ = Describe("controller", func() { Expect(result).To(Equal(reconcile.Result{Requeue: true})) }) - It("should not recover panic if RecoverPanic is false by default", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - + It("should not recover panic if RecoverPanic is false", func(ctx SpecContext) { defer func() { Expect(recover()).ShouldNot(BeNil()) }() + ctrl.RecoverPanic = ptr.To(false) ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { var res *reconcile.Result return *res, nil @@ -104,14 +113,26 @@ var _ = Describe("controller", func() { reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) }) - It("should recover panic if RecoverPanic is true", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should recover panic if RecoverPanic is true by default", func(ctx SpecContext) { + defer func() { + Expect(recover()).To(BeNil()) + }() + // RecoverPanic defaults to true. + ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { + var res *reconcile.Result + return *res, nil + }) + _, err := ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("[recovered]")) + }) + It("should recover panic if RecoverPanic is true", func(ctx SpecContext) { defer func() { Expect(recover()).To(BeNil()) }() - ctrl.RecoverPanic = true + ctrl.RecoverPanic = ptr.To(true) ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { var res *reconcile.Result return *res, nil @@ -121,77 +142,135 @@ var _ = Describe("controller", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("[recovered]")) }) + + It("should time out if ReconciliationTimeout is set", func(ctx SpecContext) { + ctrl.ReconciliationTimeout = time.Duration(1) // One nanosecond + ctrl.Do = reconcile.Func(func(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + <-ctx.Done() + return reconcile.Result{}, ctx.Err() + }) + _, err := ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + Expect(err).To(HaveOccurred()) + Expect(err).To(Equal(context.DeadlineExceeded)) + }) + + It("should not configure a timeout if ReconciliationTimeout is zero", func(ctx SpecContext) { + ctrl.Do = reconcile.Func(func(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + defer GinkgoRecover() + + _, ok := ctx.Deadline() + Expect(ok).To(BeFalse()) + return reconcile.Result{}, nil + }) + _, err := ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + Expect(err).NotTo(HaveOccurred()) + }) }) Describe("Start", func() { - It("should return an error if there is an error waiting for the informers", func() { + It("should return an error if there is an error waiting for the informers", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = time.Second f := false - ctrl.startWatches = []watchDescription{{ - src: source.NewKindWithCache(&corev1.Pod{}, &informertest.FakeInformers{Synced: &f}), - }} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), + } ctrl.Name = "foo" - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() err := ctrl.Start(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to wait for foo caches to sync")) }) - It("should error when cache sync timeout occurs", func() { - ctrl.CacheSyncTimeout = 10 * time.Nanosecond - + It("should error when cache sync timeout occurs", func(ctx SpecContext) { c, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) c = &cacheWithIndefinitelyBlockingGetInformer{c} - ctrl.startWatches = []watchDescription{{ - src: source.NewKindWithCache(&appsv1.Deployment{}, c), - }} - ctrl.Name = "testcontroller" + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(c, &appsv1.Deployment{}, &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{}), + } + ctrl.Name = testControllerName - err = ctrl.Start(context.TODO()) + err = ctrl.Start(ctx) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync: timed out waiting for cache to be synced")) + Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync kind source: *v1.Deployment: timed out waiting for cache to be synced")) }) - It("should not error when cache sync timeout is of sufficiently high", func() { + It("should not error when controller Start context is cancelled during Sources WaitForSync", func(specCtx SpecContext) { ctrl.CacheSyncTimeout = 1 * time.Second - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - sourceSynced := make(chan struct{}) c, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) - ctrl.startWatches = []watchDescription{{ - src: &singnallingSourceWrapper{ - SyncingSource: source.NewKindWithCache(&appsv1.Deployment{}, c), + c = &cacheWithIndefinitelyBlockingGetInformer{c} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), cacheSyncDone: sourceSynced, }, - }} + } + ctrl.Name = testControllerName + ctx, cancel := context.WithCancel(specCtx) go func() { defer GinkgoRecover() - Expect(c.Start(ctx)).To(Succeed()) + err = ctrl.Start(ctx) + Expect(err).To(Succeed()) }() + cancel() + <-sourceSynced + }) + + It("should error when Start() is blocking forever", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = time.Second + + controllerDone := make(chan struct{}) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-controllerDone + return ctx.Err() + })} + + ctx, cancel := context.WithTimeout(specCtx, 10*time.Second) + defer cancel() + + err := ctrl.Start(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Please ensure that its Start() method is non-blocking")) + + close(controllerDone) + }) + + It("should not error when cache sync timeout is of sufficiently high", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + + sourceSynced := make(chan struct{}) + c := &informertest.FakeInformers{} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), + cacheSyncDone: sourceSynced, + }, + } + go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).To(Succeed()) }() <-sourceSynced - }, 10.0) + }) - It("should process events from source.Channel", func() { + It("should process events from source.Channel", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second // channel to be closed when event is processed processed := make(chan struct{}) - // source channel to be injected + // source channel ch := make(chan event.GenericEvent, 1) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - // event to be sent to the channel p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, @@ -200,22 +279,20 @@ var _ = Describe("controller", func() { Object: p, } - ins := &source.Channel{Source: ch} - ins.DestBufferSize = 1 - Expect(inject.StopChannelInto(ctx.Done(), ins)).To(BeTrue()) - - // send the event to the channel - ch <- evt - - ctrl.startWatches = []watchDescription{{ - src: ins, - handler: handler.Funcs{ - GenericFunc: func(evt event.GenericEvent, q workqueue.RateLimitingInterface) { + ins := source.Channel( + ch, + handler.Funcs{ + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() close(processed) }, }, - }} + ) + + // send the event to the channel + ch <- evt + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} go func() { defer GinkgoRecover() @@ -224,212 +301,295 @@ var _ = Describe("controller", func() { <-processed }) - It("should error when channel is passed as a source but stop channel is not injected", func() { - ch := make(chan event.GenericEvent) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - ins := &source.Channel{Source: ch} - ctrl.startWatches = []watchDescription{{ - src: ins, - }} - - e := ctrl.Start(ctx) - - Expect(e).NotTo(BeNil()) - Expect(e.Error()).To(ContainSubstring("must call InjectStop on Channel before calling Start")) - }) - - It("should error when channel source is not specified", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ins := &source.Channel{} - Expect(inject.StopChannelInto(make(<-chan struct{}), ins)).To(BeTrue()) + It("should error when channel source is not specified", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second - ctrl.startWatches = []watchDescription{{ - src: &source.Channel{}, - }} + ins := source.Channel[string](nil, nil) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} e := ctrl.Start(ctx) - Expect(e).NotTo(BeNil()) + Expect(e).To(HaveOccurred()) Expect(e.Error()).To(ContainSubstring("must specify Channel.Source")) }) - It("should call Start on sources with the appropriate EventHandler, Queue, and Predicates", func() { - pr1 := &predicate.Funcs{} - pr2 := &predicate.Funcs{} - evthdl := &handler.EnqueueRequestForObject{} + It("should call Start on sources with the appropriate EventHandler, Queue, and Predicates", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second started := false - src := source.Func(func(ctx context.Context, e handler.EventHandler, q workqueue.RateLimitingInterface, p ...predicate.Predicate) error { + ctx, cancel := context.WithCancel(specCtx) + src := source.Func(func(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { defer GinkgoRecover() - Expect(e).To(Equal(evthdl)) Expect(q).To(Equal(ctrl.Queue)) - Expect(p).To(ConsistOf(pr1, pr2)) started = true + cancel() // Cancel the context so ctrl.Start() doesn't block forever return nil }) - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).NotTo(HaveOccurred()) + Expect(ctrl.Watch(src)).NotTo(HaveOccurred()) - // Use a cancelled context so Start doesn't block - ctx, cancel := context.WithCancel(context.Background()) - cancel() - Expect(ctrl.Start(ctx)).To(Succeed()) + err := ctrl.Start(ctx) + Expect(err).To(Succeed()) Expect(started).To(BeTrue()) }) - It("should return an error if there is an error starting sources", func() { + It("should return an error if there is an error starting sources", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second err := fmt.Errorf("Expected Error: could not start source") - src := source.Func(func(context.Context, handler.EventHandler, - workqueue.RateLimitingInterface, - ...predicate.Predicate) error { + src := source.Func(func(context.Context, + workqueue.TypedRateLimitingInterface[reconcile.Request], + ) error { defer GinkgoRecover() return err }) - Expect(ctrl.Watch(src, &handler.EnqueueRequestForObject{})).To(Succeed()) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + Expect(ctrl.Watch(src)).To(Succeed()) Expect(ctrl.Start(ctx)).To(Equal(err)) }) - It("should return an error if it gets started more than once", func() { + It("should return an error if it gets started more than once", func(specCtx SpecContext) { // Use a cancelled context so Start doesn't block - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() - Expect(ctrl.Start(ctx)).To(BeNil()) + Expect(ctrl.Start(ctx)).To(Succeed()) err := ctrl.Start(ctx) - Expect(err).NotTo(BeNil()) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("controller was started more than once. This is likely to be caused by being added to a manager multiple times")) }) + It("should check for correct TypedSyncingSource if custom types are used", func(specCtx SpecContext) { + queue := &priorityQueueWrapper[TestRequest]{ + TypedRateLimitingInterface: &controllertest.TypedQueue[TestRequest]{ + TypedInterface: workqueue.NewTyped[TestRequest](), + }} + ctrl := New[TestRequest](Options[TestRequest]{ + NewQueue: func(string, workqueue.TypedRateLimiter[TestRequest]) workqueue.TypedRateLimitingInterface[TestRequest] { + return queue + }, + LogConstructor: func(*TestRequest) logr.Logger { + return log.RuntimeLog.WithName("controller").WithName("test") + }, + }) + ctrl.CacheSyncTimeout = time.Second + src := &bisignallingSource[TestRequest]{ + startCall: make(chan workqueue.TypedRateLimitingInterface[TestRequest]), + startDone: make(chan error, 1), + waitCall: make(chan struct{}), + waitDone: make(chan error, 1), + } + ctrl.startWatches = []source.TypedSource[TestRequest]{src} + ctrl.Name = "foo" + ctx, cancel := context.WithCancel(specCtx) + defer cancel() + startCh := make(chan error) + go func() { + defer GinkgoRecover() + startCh <- ctrl.Start(ctx) + }() + Eventually(src.startCall).Should(Receive(Equal(queue))) + src.startDone <- nil + Eventually(src.waitCall).Should(BeClosed()) + src.waitDone <- nil + cancel() + Eventually(startCh).Should(Receive(Succeed())) + }) }) - Describe("Watch", func() { - It("should inject dependencies into the Source", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - found := false - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == src { - found = true - } - return nil - } - Expect(ctrl.Watch(src, evthdl)).NotTo(HaveOccurred()) - Expect(found).To(BeTrue(), "Source not injected") + Describe("startEventSourcesAndQueueLocked", func() { + It("should return nil when no sources are provided", func(ctx SpecContext) { + ctrl.startWatches = []source.TypedSource[reconcile.Request]{} + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) }) - It("should return an error if there is an error injecting into the Source", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - expected := fmt.Errorf("expect fail source") - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == src { - return expected - } - return nil - } - Expect(ctrl.Watch(src, evthdl)).To(Equal(expected)) + It("should initialize controller queue when called", func(ctx SpecContext) { + ctrl.startWatches = []source.TypedSource[reconcile.Request]{} + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctrl.Queue).NotTo(BeNil()) }) - It("should inject dependencies into the EventHandler", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - found := false - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == evthdl { - found = true - } - return nil + It("should return an error if a source fails to start", func(ctx SpecContext) { + expectedErr := fmt.Errorf("failed to start source") + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + // Return the error immediately so we don't get a timeout + return expectedErr + }) + + // Set a sufficiently long timeout to avoid timeouts interfering with the error being returned + ctrl.CacheSyncTimeout = 5 * time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).To(Equal(expectedErr)) + }) + + It("should return an error if a source fails to sync", func(ctx SpecContext) { + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: ptr.To(false)}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), } - Expect(ctrl.Watch(src, evthdl)).NotTo(HaveOccurred()) - Expect(found).To(BeTrue(), "EventHandler not injected") + ctrl.Name = "test-controller" + ctrl.CacheSyncTimeout = 5 * time.Second + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to wait for test-controller caches to sync")) }) - It("should return an error if there is an error injecting into the EventHandler", func() { - src := &source.Kind{Type: &corev1.Pod{}} - evthdl := &handler.EnqueueRequestForObject{} - expected := fmt.Errorf("expect fail eventhandler") - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == evthdl { - return expected - } - return nil + It("should not return an error when sources start and sync successfully", func(ctx SpecContext) { + // Create a source that starts and syncs successfully + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: ptr.To(true)}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), } - Expect(ctrl.Watch(src, evthdl)).To(Equal(expected)) + ctrl.Name = "test-controller" + ctrl.CacheSyncTimeout = 5 * time.Second + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) }) - PIt("should inject dependencies into the Reconciler", func() { - // TODO(community): Write this + It("should not return an error when context is cancelled during source sync", func(ctx SpecContext) { + sourceCtx, sourceCancel := context.WithCancel(ctx) + defer sourceCancel() + + ctrl.CacheSyncTimeout = 5 * time.Second + + // Create a bisignallingSource to control the test flow + src := &bisignallingSource[reconcile.Request]{ + startCall: make(chan workqueue.TypedRateLimitingInterface[reconcile.Request]), + startDone: make(chan error, 1), + waitCall: make(chan struct{}), + waitDone: make(chan error, 1), + } + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + + // Start the sources in a goroutine + startErrCh := make(chan error) + go func() { + defer GinkgoRecover() + startErrCh <- ctrl.startEventSourcesAndQueueLocked(sourceCtx) + }() + + // Allow source to start successfully + Eventually(src.startCall).Should(Receive()) + src.startDone <- nil + + // Wait for WaitForSync to be called + Eventually(src.waitCall).Should(BeClosed()) + + // Return context.Canceled from WaitForSync + src.waitDone <- context.Canceled + + // Also cancel the context + sourceCancel() + + // We expect to receive the context.Canceled error + err := <-startErrCh + Expect(err).To(MatchError(context.Canceled)) }) - PIt("should return an error if there is an error injecting into the Reconciler", func() { - // TODO(community): Write this + It("should timeout if source Start blocks for too long", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Millisecond + + // Create a source that blocks forever in Start + blockingSrc := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-ctx.Done() + return ctx.Err() + }) + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{blockingSrc} + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("timed out waiting for source")) }) - It("should inject dependencies into all of the Predicates", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - pr1 := &predicate.Funcs{} - pr2 := &predicate.Funcs{} - found1 := false - found2 := false - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == pr1 { - found1 = true - } - if i == pr2 { - found2 = true - } + It("should only start sources once when called multiple times concurrently", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Millisecond + + var startCount atomic.Int32 + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + startCount.Add(1) return nil + }) + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + + By("Calling startEventSourcesAndQueueLocked multiple times in parallel") + var wg sync.WaitGroup + for i := 1; i <= 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := ctrl.startEventSourcesAndQueueLocked(ctx) + // All calls should return the same nil error + Expect(err).NotTo(HaveOccurred()) + }() } - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).NotTo(HaveOccurred()) - Expect(found1).To(BeTrue(), "First Predicated not injected") - Expect(found2).To(BeTrue(), "Second Predicated not injected") + + wg.Wait() + Expect(startCount.Load()).To(Equal(int32(1)), "Source should only be started once even when called multiple times") }) - It("should return an error if there is an error injecting into any of the Predicates", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - pr1 := &predicate.Funcs{} - pr2 := &predicate.Funcs{} - expected := fmt.Errorf("expect fail predicate") - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == pr1 { - return expected - } + It("should block subsequent calls from returning until the first call to startEventSourcesAndQueueLocked has returned", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 5 * time.Second + + // finishSourceChan is closed to unblock startEventSourcesAndQueueLocked from returning + finishSourceChan := make(chan struct{}) + + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-finishSourceChan return nil - } - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).To(Equal(expected)) + }) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} - ctrl.SetFields = func(i interface{}) error { + By("Calling startEventSourcesAndQueueLocked asynchronously") + wg := sync.WaitGroup{} + go func() { defer GinkgoRecover() - if i == pr2 { - return expected - } + defer wg.Done() + + wg.Add(1) + Expect(ctrl.startEventSourcesAndQueueLocked(ctx)).To(Succeed()) + }() + + By("Calling startEventSourcesAndQueueLocked again") + var didSubsequentCallComplete atomic.Bool + go func() { + defer GinkgoRecover() + defer wg.Done() + + wg.Add(1) + Expect(ctrl.startEventSourcesAndQueueLocked(ctx)).To(Succeed()) + didSubsequentCallComplete.Store(true) + }() + + // Assert that second call to startEventSourcesAndQueueLocked is blocked while source has not finished + Consistently(didSubsequentCallComplete.Load).Should(BeFalse()) + + By("Finishing source start + sync") + finishSourceChan <- struct{}{} + + // Assert that second call to startEventSourcesAndQueueLocked is now complete + Eventually(didSubsequentCallComplete.Load).Should(BeTrue(), "startEventSourcesAndQueueLocked should complete after source is started and synced") + wg.Wait() + }) + + It("should reset c.startWatches to nil after returning and startedEventSourcesAndQueue", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Millisecond + + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { return nil - } - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).To(Equal(expected)) + }) + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctrl.startWatches).To(BeNil(), "startWatches should be reset to nil after returning") + Expect(ctrl.startedEventSourcesAndQueue).To(BeTrue(), "startedEventSourcesAndQueue should be set to true after startEventSourcesAndQueueLocked returns without error") }) }) Describe("Processing queue items from a Controller", func() { - It("should call Reconciler if an item is enqueued", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should call Reconciler if an item is enqueued", func(ctx SpecContext) { go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -445,36 +605,31 @@ var _ = Describe("controller", func() { Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) }) - It("should continue to process additional queue items after the first", func() { - ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { - defer GinkgoRecover() - Fail("Reconciler should not have been called") - return reconcile.Result{}, nil - }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should requeue a Request if there is an error and continue processing items", func(ctx SpecContext) { go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) }() - By("adding two bad items to the queue") - queue.Add("foo/bar1") - queue.Add("foo/bar2") + queue.Add(request) - By("expecting both of them to be skipped") - Eventually(queue.Len).Should(Equal(0)) - Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) - }) + By("Invoking Reconciler which will give an error") + fakeReconcile.AddResult(reconcile.Result{}, fmt.Errorf("expected error: reconcile")) + Expect(<-reconciled).To(Equal(request)) + queue.AddedRateLimitedLock.Lock() + Expect(queue.AddedRatelimited).To(Equal([]any{request})) + queue.AddedRateLimitedLock.Unlock() - PIt("should forget an item if it is not a Request and continue processing items", func() { - // TODO(community): write this test - }) + By("Invoking Reconciler a second time without error") + fakeReconcile.AddResult(reconcile.Result{}, nil) + Expect(<-reconciled).To(Equal(request)) - It("should requeue a Request if there is an error and continue processing items", func() { + By("Removing the item from the queue") + Eventually(queue.Len).Should(Equal(0)) + Eventually(func() int { return queue.NumRequeues(request) }, 1.0).Should(Equal(0)) + }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should not requeue a Request if there is a terminal error", func(ctx SpecContext) { go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -483,26 +638,24 @@ var _ = Describe("controller", func() { queue.Add(request) By("Invoking Reconciler which will give an error") - fakeReconcile.AddResult(reconcile.Result{}, fmt.Errorf("expected error: reconcile")) + fakeReconcile.AddResult(reconcile.Result{}, reconcile.TerminalError(fmt.Errorf("expected error: reconcile"))) Expect(<-reconciled).To(Equal(request)) - By("Invoking Reconciler a second time without error") - fakeReconcile.AddResult(reconcile.Result{}, nil) - Expect(<-reconciled).To(Equal(request)) + queue.AddedRateLimitedLock.Lock() + Expect(queue.AddedRatelimited).To(BeEmpty()) + queue.AddedRateLimitedLock.Unlock() - By("Removing the item from the queue") - Eventually(queue.Len).Should(Equal(0)) - Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) - }, 1.0) + Expect(queue.Len()).Should(Equal(0)) + }) // TODO(directxman12): we should ensure that backoff occurrs with error requeue - It("should not reset backoff until there's a non-error result", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should not reset backoff until there's a non-error result", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -533,12 +686,12 @@ var _ = Describe("controller", func() { Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) - It("should requeue a Request with rate limiting if the Result sets Requeue:true and continue processing items", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should requeue a Request with rate limiting if the Result sets Requeue:true and continue processing items", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -563,12 +716,70 @@ var _ = Describe("controller", func() { Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) - It("should requeue a Request after a duration (but not rate-limitted) if the Result sets RequeueAfter (regardless of Requeue)", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should retain the priority when the reconciler requests a requeue", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will request a requeue") + fakeReconcile.AddResult(reconcile.Result{Requeue: true}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(10), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should use the priority from Result when the reconciler requests a requeue", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will request a requeue") + fakeReconcile.AddResult(reconcile.Result{Requeue: true, Priority: ptr.To(99)}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(99), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should requeue a Request after a duration (but not rate-limited) if the Result sets RequeueAfter (regardless of Requeue)", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -593,35 +804,151 @@ var _ = Describe("controller", func() { Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) - It("should perform error behavior if error is not nil, regardless of RequeueAfter", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should retain the priority with RequeueAfter", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) }() - dq.Add(request) - Expect(dq.getCounts()).To(Equal(countInfo{Trying: 1})) - - By("Invoking Reconciler which will ask for requeueafter with an error") - fakeReconcile.AddResult(reconcile.Result{RequeueAfter: time.Millisecond * 100}, fmt.Errorf("expected error: reconcile")) - Expect(<-reconciled).To(Equal(request)) - Eventually(dq.getCounts).Should(Equal(countInfo{Trying: 1, AddRateLimited: 1})) + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) - By("Invoking Reconciler a second time asking for requeueafter without errors") + By("Invoking Reconciler which will ask for RequeueAfter") fakeReconcile.AddResult(reconcile.Result{RequeueAfter: time.Millisecond * 100}, nil) Expect(<-reconciled).To(Equal(request)) - Eventually(dq.getCounts).Should(Equal(countInfo{AddAfter: 1, AddRateLimited: 1})) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + After: time.Millisecond * 100, + Priority: ptr.To(10), + }, + items: []reconcile.Request{request}, + }})) + }) - By("Removing the item from the queue") + It("should use the priority from Result with RequeueAfter", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will ask for RequeueAfter") + fakeReconcile.AddResult(reconcile.Result{RequeueAfter: time.Millisecond * 100, Priority: ptr.To(99)}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + After: time.Millisecond * 100, + Priority: ptr.To(99), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should perform error behavior if error is not nil, regardless of RequeueAfter", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + dq.Add(request) + Expect(dq.getCounts()).To(Equal(countInfo{Trying: 1})) + + By("Invoking Reconciler which will ask for requeueafter with an error") + fakeReconcile.AddResult(reconcile.Result{RequeueAfter: time.Millisecond * 100}, fmt.Errorf("expected error: reconcile")) + Expect(<-reconciled).To(Equal(request)) + Eventually(dq.getCounts).Should(Equal(countInfo{Trying: 1, AddRateLimited: 1})) + + By("Invoking Reconciler a second time asking for requeueafter without errors") + fakeReconcile.AddResult(reconcile.Result{RequeueAfter: time.Millisecond * 100}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(dq.getCounts).Should(Equal(countInfo{AddAfter: 1, AddRateLimited: 1})) + + By("Removing the item from the queue") Eventually(dq.Len).Should(Equal(0)) Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) + It("should retain the priority when there was an error", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will return an error") + fakeReconcile.AddResult(reconcile.Result{}, errors.New("oups, I did it again")) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(10), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should use the priority from Result when there was an error", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will return an error") + fakeReconcile.AddResult(reconcile.Result{Priority: ptr.To(99)}, errors.New("oups, I did it again")) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(99), + }, + items: []reconcile.Request{request}, + }})) + }) + PIt("should return if the queue is shutdown", func() { // TODO(community): write this test }) @@ -642,7 +969,7 @@ var _ = Describe("controller", func() { reconcileTotal.Reset() }) - It("should get updated on successful reconciliation", func() { + It("should get updated on successful reconciliation", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "success").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -651,8 +978,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -669,9 +994,9 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) - }, 2.0) + }) - It("should get updated on reconcile errors", func() { + It("should get updated on reconcile errors", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "error").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -680,8 +1005,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -698,9 +1021,9 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) - }, 2.0) + }) - It("should get updated when reconcile returns with retry enabled", func() { + It("should get updated when reconcile returns with retry enabled", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -709,8 +1032,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -728,9 +1049,9 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) - }, 2.0) + }) - It("should get updated when reconcile returns with retryAfter enabled", func() { + It("should get updated when reconcile returns with retryAfter enabled", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry_after").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -739,8 +1060,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -757,11 +1076,11 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) - }, 2.0) + }) }) Context("should update prometheus metrics", func() { - It("should requeue a Request if there is an error and continue processing items", func() { + It("should requeue a Request if there is an error and continue processing items", func(ctx SpecContext) { var reconcileErrs dto.Metric ctrlmetrics.ReconcileErrors.Reset() Expect(func() error { @@ -772,8 +1091,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -798,9 +1115,9 @@ var _ = Describe("controller", func() { By("Removing the item from the queue") Eventually(queue.Len).Should(Equal(0)) Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) - }, 2.0) + }) - It("should add a reconcile time to the reconcile time histogram", func() { + It("should add a reconcile time to the reconcile time histogram", func(ctx SpecContext) { var reconcileTime dto.Metric ctrlmetrics.ReconcileTime.Reset() @@ -814,8 +1131,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -839,13 +1154,538 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) - }, 4.0) + }) + }) + }) + + Describe("Warmup", func() { + JustBeforeEach(func() { + ctrl.EnableWarmup = ptr.To(true) + }) + + It("should track warmup status correctly with successful sync", func(ctx SpecContext) { + // Setup controller with sources that complete successfully + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return nil + }), + } + + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }) + + It("should return an error if there is an error waiting for the informers", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: ptr.To(false)}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), + } + ctrl.Name = testControllerName + err := ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync")) + }) + + It("should error when cache sync timeout occurs", func(ctx SpecContext) { + c, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + c = &cacheWithIndefinitelyBlockingGetInformer{c} + + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(c, &appsv1.Deployment{}, &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{}), + } + ctrl.Name = testControllerName + + err = ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync kind source: *v1.Deployment: timed out waiting for cache to be synced")) + }) + + It("should not error when controller Warmup context is cancelled during Sources WaitForSync", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Second + + sourceSynced := make(chan struct{}) + c, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + c = &cacheWithIndefinitelyBlockingGetInformer{c} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), + cacheSyncDone: sourceSynced, + }, + } + ctrl.Name = testControllerName + + ctx, cancel := context.WithCancel(specCtx) + go func() { + defer GinkgoRecover() + err = ctrl.Warmup(ctx) + Expect(err).To(Succeed()) + }() + + cancel() + <-sourceSynced + }) + + It("should error when Warmup() is blocking forever", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = time.Second + + controllerDone := make(chan struct{}) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-controllerDone + return ctx.Err() + })} + + ctx, cancel := context.WithTimeout(specCtx, 10*time.Second) + defer cancel() + + err := ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Please ensure that its Start() method is non-blocking")) + + close(controllerDone) + }) + + It("should not error when cache sync timeout is of sufficiently high", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + + sourceSynced := make(chan struct{}) + c := &informertest.FakeInformers{} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), + cacheSyncDone: sourceSynced, + }, + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + + <-sourceSynced + }) + + It("should process events from source.Channel", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + // channel to be closed when event is processed + processed := make(chan struct{}) + // source channel + ch := make(chan event.GenericEvent, 1) + + // event to be sent to the channel + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, + } + evt := event.GenericEvent{ + Object: p, + } + + ins := source.Channel( + ch, + handler.Funcs{ + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + close(processed) + }, + }, + ) + + // send the event to the channel + ch <- evt + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} + + go func() { + defer GinkgoRecover() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + <-processed + }) + + It("should error when channel source is not specified", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + + ins := source.Channel[string](nil, nil) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} + + e := ctrl.Warmup(ctx) + Expect(e).To(HaveOccurred()) + Expect(e.Error()).To(ContainSubstring("must specify Channel.Source")) + }) + + It("should call Start on sources with the appropriate EventHandler, Queue, and Predicates", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + started := false + ctx, cancel := context.WithCancel(specCtx) + src := source.Func(func(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + defer GinkgoRecover() + Expect(q).To(Equal(ctrl.Queue)) + + started = true + cancel() // Cancel the context so ctrl.Warmup() doesn't block forever + return nil + }) + Expect(ctrl.Watch(src)).NotTo(HaveOccurred()) + + err := ctrl.Warmup(ctx) + Expect(err).To(Succeed()) + Expect(started).To(BeTrue()) + }) + + It("should return an error if there is an error starting sources", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + err := fmt.Errorf("Expected Error: could not start source") + src := source.Func(func(context.Context, + workqueue.TypedRateLimitingInterface[reconcile.Request], + ) error { + defer GinkgoRecover() + return err + }) + Expect(ctrl.Watch(src)).To(Succeed()) + + Expect(ctrl.Warmup(ctx)).To(Equal(err)) + }) + + It("should track warmup status correctly with unsuccessful sync", func(ctx SpecContext) { + // Setup controller with sources that complete with error + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return errors.New("sync error") + }), + } + + err := ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sync error")) + }) + + It("should call Start on sources with the appropriate non-nil queue", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + started := false + ctx, cancel := context.WithCancel(specCtx) + src := source.Func(func(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + defer GinkgoRecover() + Expect(q).ToNot(BeNil()) + Expect(q).To(Equal(ctrl.Queue)) + + started = true + cancel() // Cancel the context so ctrl.Start() doesn't block forever + return nil + }) + Expect(ctrl.Watch(src)).To(Succeed()) + Expect(ctrl.Warmup(ctx)).To(Succeed()) + Expect(ctrl.Queue).ToNot(BeNil()) + Expect(started).To(BeTrue()) + }) + + It("should return true if context is cancelled while waiting for source to start", func(specCtx SpecContext) { + // Setup controller with sources that complete with error + ctx, cancel := context.WithCancel(specCtx) + defer cancel() + + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-ctx.Done() + return nil + }), + } + + // channel to prevent the goroutine from outliving the It test + waitChan := make(chan struct{}) + + // Invoked in a goroutine because Warmup will block + go func() { + defer GinkgoRecover() + defer close(waitChan) + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + + cancel() + <-waitChan + }) + + It("should be called before leader election runnables if warmup is enabled", func(specCtx SpecContext) { + // This unit test exists to ensure that a warmup enabled controller will actually be + // called in the warmup phase before the leader election runnables are started. It + // catches regressions in the controller that would not implement warmupRunnable from + // pkg/manager. + ctx, cancel := context.WithCancel(specCtx) + + By("Creating a channel to track execution order") + runnableExecutionOrderChan := make(chan string, 2) + const nonWarmupRunnableName = "nonWarmupRunnable" + const warmupRunnableName = "warmupRunnable" + + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + runnableExecutionOrderChan <- warmupRunnableName + return nil + }), + } + + nonWarmupCtrl := New[reconcile.Request](Options[reconcile.Request]{ + MaxConcurrentReconciles: 1, + Do: fakeReconcile, + NewQueue: func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return queue + }, + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return log.RuntimeLog.WithName("controller").WithName("test") + }, + CacheSyncTimeout: time.Second, + EnableWarmup: ptr.To(false), + LeaderElected: ptr.To(true), + }) + nonWarmupCtrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + runnableExecutionOrderChan <- nonWarmupRunnableName + return nil + }), + } + + By("Creating a test resource lock with hooks") + resourceLock, err := fakeleaderelection.NewResourceLock(nil, nil, leaderelection.Options{}) + Expect(err).ToNot(HaveOccurred()) + + By("Creating a manager") + testenv = &envtest.Environment{} + cfg, err := testenv.Start() + Expect(err).NotTo(HaveOccurred()) + m, err := manager.New(cfg, manager.Options{ + LeaderElection: true, + LeaderElectionID: "some-leader-election-id", + LeaderElectionNamespace: "default", + LeaderElectionResourceLockInterface: resourceLock, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Adding warmup and non-warmup controllers to the manager") + Expect(m.Add(ctrl)).To(Succeed()) + Expect(m.Add(nonWarmupCtrl)).To(Succeed()) + + By("Blocking leader election") + resourceLockWithHooks, ok := resourceLock.(fakeleaderelection.ControllableResourceLockInterface) + Expect(ok).To(BeTrue(), "resource lock should implement ResourceLockInterfaceWithHooks") + resourceLockWithHooks.BlockLeaderElection() + + By("Starting the manager") + waitChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(waitChan) + Expect(m.Start(ctx)).To(Succeed()) + }() + Expect(<-runnableExecutionOrderChan).To(Equal(warmupRunnableName)) + + By("Unblocking leader election") + resourceLockWithHooks.UnblockLeaderElection() + <-m.Elected() + Expect(<-runnableExecutionOrderChan).To(Equal(nonWarmupRunnableName)) + + cancel() + <-waitChan + }) + + It("should not cause a data race when called concurrently", func(ctx SpecContext) { + + ctrl.CacheSyncTimeout = time.Second + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return nil + }), + } + + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + } + + wg.Wait() + }) + + It("should not cause a data race when called concurrently with Start and only start sources once", func(specCtx SpecContext) { + ctx, cancel := context.WithCancel(specCtx) + + ctrl.CacheSyncTimeout = time.Second + numWatches := 10 + + var watchStartedCount atomic.Int32 + for range numWatches { + ctrl.startWatches = append(ctrl.startWatches, source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + watchStartedCount.Add(1) + return nil + })) + } + + By("calling Warmup and Start concurrently") + blockOnStartChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).To(Succeed()) + close(blockOnStartChan) + }() + + blockOnWarmupChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + close(blockOnWarmupChan) + }() + + <-blockOnWarmupChan + + cancel() + + <-blockOnStartChan + + Expect(watchStartedCount.Load()).To(Equal(int32(numWatches)), "source should only be started once") + Expect(ctrl.startWatches).To(BeNil(), "startWatches should be reset to nil after they are started") + }) + + It("should start sources added after Warmup is called", func(specCtx SpecContext) { + ctx, cancel := context.WithCancel(specCtx) + + ctrl.CacheSyncTimeout = time.Second + + Expect(ctrl.Warmup(ctx)).To(Succeed()) + + By("starting a watch after warmup is added") + var didWatchStart atomic.Bool + Expect(ctrl.Watch(source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + didWatchStart.Store(true) + return nil + }))).To(Succeed()) + + waitChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).To(Succeed()) + close(waitChan) + }() + + Eventually(didWatchStart.Load).Should(BeTrue(), "watch should be started if it is added after Warmup") + + cancel() + <-waitChan }) + + DescribeTable("should not leak goroutines when manager is stopped with warmup runnable", + func(specContext SpecContext, leaderElection bool) { + ctx, cancel := context.WithCancel(specContext) + defer cancel() + + ctrl.CacheSyncTimeout = time.Second + + By("Creating a manager") + testenv = &envtest.Environment{} + cfg, err := testenv.Start() + Expect(err).NotTo(HaveOccurred()) + m, err := manager.New(cfg, manager.Options{ + LeaderElection: leaderElection, + LeaderElectionID: "some-leader-election-id", + LeaderElectionNamespace: "default", + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-ctx.Done() + return nil + }), + } + Expect(m.Add(ctrl)).To(Succeed()) + + // ignore needs to go after the testenv.Start() call to ignore the apiserver + // process + currentGRs := goleak.IgnoreCurrent() + waitChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).To(Succeed()) + close(waitChan) + }() + + <-m.Elected() + By("stopping the manager via context") + cancel() + + Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) + <-waitChan + }, + Entry("and with leader election enabled", true), + Entry("and without leader election enabled", false), + ) + }) + + Describe("Warmup with warmup disabled", func() { + JustBeforeEach(func() { + ctrl.EnableWarmup = ptr.To(false) + }) + + It("should not start sources when Warmup is called if warmup is disabled but start it when Start is called.", func(specCtx SpecContext) { + // Setup controller with sources that complete successfully + ctx, cancel := context.WithCancel(specCtx) + + ctrl.CacheSyncTimeout = time.Second + var isSourceStarted atomic.Bool + isSourceStarted.Store(false) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + isSourceStarted.Store(true) + return nil + }), + } + + By("Calling Warmup when EnableWarmup is false") + err := ctrl.Warmup(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(isSourceStarted.Load()).To(BeFalse()) + + By("Calling Start when EnableWarmup is false") + waitChan := make(chan struct{}) + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).To(Succeed()) + close(waitChan) + }() + Eventually(isSourceStarted.Load).Should(BeTrue()) + cancel() + <-waitChan + }) + }) +}) + +var _ = Describe("ReconcileIDFromContext function", func() { + It("should return an empty string if there is nothing in the context", func(ctx SpecContext) { + reconcileID := ReconcileIDFromContext(ctx) + + Expect(reconcileID).To(Equal(types.UID(""))) + }) + + It("should return the correct reconcileID from context", func(specContext SpecContext) { + const expectedReconcileID = types.UID("uuid") + ctx := addReconcileID(specContext, expectedReconcileID) + reconcileID := ReconcileIDFromContext(ctx) + + Expect(reconcileID).To(Equal(expectedReconcileID)) }) }) type DelegatingQueue struct { - workqueue.RateLimitingInterface + workqueue.TypedRateLimitingInterface[reconcile.Request] mu sync.Mutex countAddRateLimited int @@ -853,36 +1693,36 @@ type DelegatingQueue struct { countAddAfter int } -func (q *DelegatingQueue) AddRateLimited(item interface{}) { +func (q *DelegatingQueue) AddRateLimited(item reconcile.Request) { q.mu.Lock() defer q.mu.Unlock() q.countAddRateLimited++ - q.RateLimitingInterface.AddRateLimited(item) + q.TypedRateLimitingInterface.AddRateLimited(item) } -func (q *DelegatingQueue) AddAfter(item interface{}, d time.Duration) { +func (q *DelegatingQueue) AddAfter(item reconcile.Request, d time.Duration) { q.mu.Lock() defer q.mu.Unlock() q.countAddAfter++ - q.RateLimitingInterface.AddAfter(item, d) + q.TypedRateLimitingInterface.AddAfter(item, d) } -func (q *DelegatingQueue) Add(item interface{}) { +func (q *DelegatingQueue) Add(item reconcile.Request) { q.mu.Lock() defer q.mu.Unlock() q.countAdd++ - q.RateLimitingInterface.Add(item) + q.TypedRateLimitingInterface.Add(item) } -func (q *DelegatingQueue) Forget(item interface{}) { +func (q *DelegatingQueue) Forget(item reconcile.Request) { q.mu.Lock() defer q.mu.Unlock() q.countAdd-- - q.RateLimitingInterface.Forget(item) + q.TypedRateLimitingInterface.Forget(item) } type countInfo struct { @@ -927,6 +1767,15 @@ type singnallingSourceWrapper struct { source.SyncingSource } +func (s *singnallingSourceWrapper) Start(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + err := s.SyncingSource.Start(ctx, q) + if err != nil { + // WaitForSync will never be called if this errors, so close the channel to prevent deadlocks in tests + close(s.cacheSyncDone) + } + return err +} + func (s *singnallingSourceWrapper) WaitForSync(ctx context.Context) error { defer func() { close(s.cacheSyncDone) @@ -946,7 +1795,62 @@ type cacheWithIndefinitelyBlockingGetInformer struct { cache.Cache } -func (c *cacheWithIndefinitelyBlockingGetInformer) GetInformer(ctx context.Context, obj client.Object) (cache.Informer, error) { +func (c *cacheWithIndefinitelyBlockingGetInformer) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { <-ctx.Done() return nil, errors.New("GetInformer timed out") } + +type bisignallingSource[T comparable] struct { + // receives the queue that is passed to Start + startCall chan workqueue.TypedRateLimitingInterface[T] + // passes an error to return from Start + startDone chan error + // closed when WaitForSync is called + waitCall chan struct{} + // passes an error to return from WaitForSync + waitDone chan error +} + +var _ source.TypedSyncingSource[int] = (*bisignallingSource[int])(nil) + +func (t *bisignallingSource[T]) Start(ctx context.Context, q workqueue.TypedRateLimitingInterface[T]) error { + select { + case t.startCall <- q: + case <-ctx.Done(): + return ctx.Err() + } + select { + case err := <-t.startDone: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (t *bisignallingSource[T]) WaitForSync(ctx context.Context) error { + close(t.waitCall) + select { + case err := <-t.waitDone: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +type priorityQueueAddition struct { + priorityqueue.AddOpts + items []reconcile.Request +} + +type fakePriorityQueue struct { + priorityqueue.PriorityQueue[reconcile.Request] + + lock sync.Mutex + added []priorityQueueAddition +} + +func (f *fakePriorityQueue) AddWithOpts(o priorityqueue.AddOpts, items ...reconcile.Request) { + f.lock.Lock() + defer f.lock.Unlock() + f.added = append(f.added, priorityQueueAddition{AddOpts: o, items: items}) +} diff --git a/pkg/internal/controller/metrics/metrics.go b/pkg/internal/controller/metrics/metrics.go index baec669277..450e9ae25b 100644 --- a/pkg/internal/controller/metrics/metrics.go +++ b/pkg/internal/controller/metrics/metrics.go @@ -17,6 +17,8 @@ limitations under the License. package metrics import ( + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -39,6 +41,20 @@ var ( Help: "Total number of reconciliation errors per controller", }, []string{"controller"}) + // TerminalReconcileErrors is a prometheus counter metrics which holds the total + // number of terminal errors from the Reconciler. + TerminalReconcileErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "controller_runtime_terminal_reconcile_errors_total", + Help: "Total number of terminal reconciliation errors per controller", + }, []string{"controller"}) + + // ReconcilePanics is a prometheus counter metrics which holds the total + // number of panics from the Reconciler. + ReconcilePanics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "controller_runtime_reconcile_panics_total", + Help: "Total number of reconciliation panics per controller", + }, []string{"controller"}) + // ReconcileTime is a prometheus metric which keeps track of the duration // of reconciliations. ReconcileTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ @@ -46,6 +62,9 @@ var ( Help: "Length of time per reconciliation per controller", Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60}, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"controller"}) // WorkerCount is a prometheus metric which holds the number of @@ -67,12 +86,14 @@ func init() { metrics.Registry.MustRegister( ReconcileTotal, ReconcileErrors, + TerminalReconcileErrors, + ReconcilePanics, ReconcileTime, WorkerCount, ActiveWorkers, // expose process metrics like CPU, Memory, file descriptor usage etc. collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - // expose Go runtime metrics like GC stats, memory stats etc. - collectors.NewGoCollector(), + // expose all Go runtime metrics like GC stats, memory stats etc. + collectors.NewGoCollector(collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll)), ) } diff --git a/pkg/internal/field/selector/utils.go b/pkg/internal/field/selector/utils.go new file mode 100644 index 0000000000..8f6dc71ede --- /dev/null +++ b/pkg/internal/field/selector/utils.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selector + +import ( + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/selection" +) + +// RequiresExactMatch checks if the given field selector is of the form `k=v` or `k==v`. +func RequiresExactMatch(sel fields.Selector) bool { + reqs := sel.Requirements() + if len(reqs) == 0 { + return false + } + + for _, req := range reqs { + if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { + return false + } + } + return true +} diff --git a/pkg/patterns/operator/doc.go b/pkg/internal/field/selector/utils_suite_test.go similarity index 67% rename from pkg/patterns/operator/doc.go rename to pkg/internal/field/selector/utils_suite_test.go index 5ccd0791af..dd42f1d1ac 100644 --- a/pkg/patterns/operator/doc.go +++ b/pkg/internal/field/selector/utils_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,10 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Package operator serves to redirect users to the application package. +package selector_test -Operators are the common name for Kubernetes APIs which manage specific applications. e.g. Spark Operator, -Etcd Operator. -*/ -package operator +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSource(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Fields Selector Utils Suite") +} diff --git a/pkg/internal/field/selector/utils_test.go b/pkg/internal/field/selector/utils_test.go new file mode 100644 index 0000000000..a48bbf4e5a --- /dev/null +++ b/pkg/internal/field/selector/utils_test.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selector_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/fields" + + . "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" +) + +var _ = Describe("RequiresExactMatch function", func() { + + It("Returns false when the selector matches everything", func() { + requiresExactMatch := RequiresExactMatch(fields.Everything()) + Expect(requiresExactMatch).To(BeFalse()) + }) + + It("Returns false when the selector matches nothing", func() { + requiresExactMatch := RequiresExactMatch(fields.Nothing()) + Expect(requiresExactMatch).To(BeFalse()) + }) + + It("Returns false when the selector has the form key!=val", func() { + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key!=val")) + Expect(requiresExactMatch).To(BeFalse()) + }) + + It("Returns true when the selector has the form key1==val1,key2==val2", func() { + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key1==val1,key2==val2")) + Expect(requiresExactMatch).To(BeTrue()) + }) + + It("Returns true when the selector has the form key==val", func() { + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key==val")) + Expect(requiresExactMatch).To(BeTrue()) + }) + + It("Returns true when the selector has the form key=val", func() { + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key=val")) + Expect(requiresExactMatch).To(BeTrue()) + }) +}) diff --git a/pkg/internal/flock/flock_unix.go b/pkg/internal/flock/flock_unix.go index 3a904f3f5f..71ec576df2 100644 --- a/pkg/internal/flock/flock_unix.go +++ b/pkg/internal/flock/flock_unix.go @@ -1,3 +1,4 @@ +//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly // +build linux darwin freebsd openbsd netbsd dragonfly /* diff --git a/pkg/internal/httpserver/server.go b/pkg/internal/httpserver/server.go new file mode 100644 index 0000000000..b5f91f18e0 --- /dev/null +++ b/pkg/internal/httpserver/server.go @@ -0,0 +1,16 @@ +package httpserver + +import ( + "net/http" + "time" +) + +// New returns a new server with sane defaults. +func New(handler http.Handler) *http.Server { + return &http.Server{ + Handler: handler, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout + ReadHeaderTimeout: 32 * time.Second, + } +} diff --git a/pkg/internal/metrics/workqueue.go b/pkg/internal/metrics/workqueue.go new file mode 100644 index 0000000000..402319817b --- /dev/null +++ b/pkg/internal/metrics/workqueue.go @@ -0,0 +1,170 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// This file is copied and adapted from k8s.io/component-base/metrics/prometheus/workqueue +// which registers metrics to the k8s legacy Registry. We require very +// similar functionality, but must register metrics to a different Registry. + +// Metrics subsystem and all keys used by the workqueue. +const ( + WorkQueueSubsystem = metrics.WorkQueueSubsystem + DepthKey = metrics.DepthKey + AddsKey = metrics.AddsKey + QueueLatencyKey = metrics.QueueLatencyKey + WorkDurationKey = metrics.WorkDurationKey + UnfinishedWorkKey = metrics.UnfinishedWorkKey + LongestRunningProcessorKey = metrics.LongestRunningProcessorKey + RetriesKey = metrics.RetriesKey +) + +var ( + depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Subsystem: WorkQueueSubsystem, + Name: DepthKey, + Help: "Current depth of workqueue by workqueue and priority", + }, []string{"name", "controller", "priority"}) + + adds = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: WorkQueueSubsystem, + Name: AddsKey, + Help: "Total number of adds handled by workqueue", + }, []string{"name", "controller"}) + + latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Subsystem: WorkQueueSubsystem, + Name: QueueLatencyKey, + Help: "How long in seconds an item stays in workqueue before being requested", + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 12), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"name", "controller"}) + + workDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Subsystem: WorkQueueSubsystem, + Name: WorkDurationKey, + Help: "How long in seconds processing an item from workqueue takes.", + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 12), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"name", "controller"}) + + unfinished = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Subsystem: WorkQueueSubsystem, + Name: UnfinishedWorkKey, + Help: "How many seconds of work has been done that " + + "is in progress and hasn't been observed by work_duration. Large " + + "values indicate stuck threads. One can deduce the number of stuck " + + "threads by observing the rate at which this increases.", + }, []string{"name", "controller"}) + + longestRunningProcessor = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Subsystem: WorkQueueSubsystem, + Name: LongestRunningProcessorKey, + Help: "How many seconds has the longest running " + + "processor for workqueue been running.", + }, []string{"name", "controller"}) + + retries = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: WorkQueueSubsystem, + Name: RetriesKey, + Help: "Total number of retries handled by workqueue", + }, []string{"name", "controller"}) +) + +func init() { + metrics.Registry.MustRegister(depth) + metrics.Registry.MustRegister(adds) + metrics.Registry.MustRegister(latency) + metrics.Registry.MustRegister(workDuration) + metrics.Registry.MustRegister(unfinished) + metrics.Registry.MustRegister(longestRunningProcessor) + metrics.Registry.MustRegister(retries) + + workqueue.SetProvider(WorkqueueMetricsProvider{}) +} + +type WorkqueueMetricsProvider struct{} + +func (WorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { + return depth.WithLabelValues(name, name, "") // no priority +} + +func (WorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { + return adds.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { + return latency.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { + return workDuration.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { + return unfinished.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { + return longestRunningProcessor.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { + return retries.WithLabelValues(name, name) +} + +type MetricsProviderWithPriority interface { + workqueue.MetricsProvider + + NewDepthMetricWithPriority(name string) DepthMetricWithPriority +} + +// DepthMetricWithPriority represents a depth metric with priority. +type DepthMetricWithPriority interface { + Inc(priority int) + Dec(priority int) +} + +var _ MetricsProviderWithPriority = WorkqueueMetricsProvider{} + +func (WorkqueueMetricsProvider) NewDepthMetricWithPriority(name string) DepthMetricWithPriority { + return &depthWithPriorityMetric{lvs: []string{name, name}} +} + +type depthWithPriorityMetric struct { + lvs []string +} + +func (g *depthWithPriorityMetric) Inc(priority int) { + depth.WithLabelValues(append(g.lvs, strconv.Itoa(priority))...).Inc() +} + +func (g *depthWithPriorityMetric) Dec(priority int) { + depth.WithLabelValues(append(g.lvs, strconv.Itoa(priority))...).Dec() +} diff --git a/pkg/internal/objectutil/objectutil.go b/pkg/internal/objectutil/objectutil.go index 7057f3dbe4..0189c04323 100644 --- a/pkg/internal/objectutil/objectutil.go +++ b/pkg/internal/objectutil/objectutil.go @@ -17,14 +17,9 @@ limitations under the License. package objectutil import ( - "errors" - "fmt" - apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) // FilterWithLabels returns a copy of the items in objs matching labelSel. @@ -45,34 +40,3 @@ func FilterWithLabels(objs []runtime.Object, labelSel labels.Selector) ([]runtim } return outItems, nil } - -// IsAPINamespaced returns true if the object is namespace scoped. -// For unstructured objects the gvk is found from the object itself. -func IsAPINamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) { - gvk, err := apiutil.GVKForObject(obj, scheme) - if err != nil { - return false, err - } - - return IsAPINamespacedWithGVK(gvk, scheme, restmapper) -} - -// IsAPINamespacedWithGVK returns true if the object having the provided -// GVK is namespace scoped. -func IsAPINamespacedWithGVK(gk schema.GroupVersionKind, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) { - restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gk.Group, Kind: gk.Kind}) - if err != nil { - return false, fmt.Errorf("failed to get restmapping: %w", err) - } - - scope := restmapping.Scope.Name() - - if scope == "" { - return false, errors.New("scope cannot be identified, empty scope returned") - } - - if scope != apimeta.RESTScopeNameRoot { - return true, nil - } - return false, nil -} diff --git a/pkg/internal/recorder/recorder.go b/pkg/internal/recorder/recorder.go index 46cc1714bd..21f0146ba3 100644 --- a/pkg/internal/recorder/recorder.go +++ b/pkg/internal/recorder/recorder.go @@ -19,6 +19,7 @@ package recorder import ( "context" "fmt" + "net/http" "sync" "github.com/go-logr/logr" @@ -100,7 +101,7 @@ func (p *Provider) getBroadcaster() record.EventBroadcaster { broadcaster.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: p.evtClient}) broadcaster.StartEventWatcher( func(e *corev1.Event) { - p.logger.V(1).Info(e.Type, "object", e.InvolvedObject, "reason", e.Reason, "message", e.Message) + p.logger.V(1).Info(e.Message, "type", e.Type, "object", e.InvolvedObject, "reason", e.Reason) }) p.broadcaster = broadcaster p.stopBroadcaster = stop @@ -110,8 +111,12 @@ func (p *Provider) getBroadcaster() record.EventBroadcaster { } // NewProvider create a new Provider instance. -func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) { - corev1Client, err := corev1client.NewForConfig(config) +func NewProvider(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) { + if httpClient == nil { + panic("httpClient must not be nil") + } + + corev1Client, err := corev1client.NewForConfigAndClient(config, httpClient) if err != nil { return nil, fmt.Errorf("failed to init client: %w", err) } diff --git a/pkg/internal/recorder/recorder_integration_test.go b/pkg/internal/recorder/recorder_integration_test.go index 5bafaabf5a..c278fbde79 100644 --- a/pkg/internal/recorder/recorder_integration_test.go +++ b/pkg/internal/recorder/recorder_integration_test.go @@ -31,13 +31,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("recorder", func() { Describe("recorder", func() { - It("should publish events", func() { + It("should publish events", func(ctx SpecContext) { By("Creating the Manager") cm, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -56,12 +56,10 @@ var _ = Describe("recorder", func() { Expect(err).NotTo(HaveOccurred()) By("Watching Resources") - err = instance.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForObject{}) + err = instance.Watch(source.Kind(cm.GetCache(), &appsv1.Deployment{}, &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{})) Expect(err).NotTo(HaveOccurred()) By("Starting the Manager") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(cm.Start(ctx)).NotTo(HaveOccurred()) diff --git a/pkg/internal/recorder/recorder_suite_test.go b/pkg/internal/recorder/recorder_suite_test.go index ee8f98fae0..e5b5836d58 100644 --- a/pkg/internal/recorder/recorder_suite_test.go +++ b/pkg/internal/recorder/recorder_suite_test.go @@ -17,26 +17,26 @@ limitations under the License. package recorder_test import ( + "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestRecorder(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Recorder Integration Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Recorder Integration Suite") } var testenv *envtest.Environment var cfg *rest.Config +var httpClient *http.Client var clientset *kubernetes.Clientset var _ = BeforeSuite(func() { @@ -48,9 +48,12 @@ var _ = BeforeSuite(func() { cfg, err = testenv.Start() Expect(err).NotTo(HaveOccurred()) + httpClient, err = rest.HTTPClientFor(cfg) + Expect(err).ToNot(HaveOccurred()) + clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/internal/recorder/recorder_test.go b/pkg/internal/recorder/recorder_test.go index 08d60f9deb..e226e165a3 100644 --- a/pkg/internal/recorder/recorder_test.go +++ b/pkg/internal/recorder/recorder_test.go @@ -18,7 +18,7 @@ package recorder_test import ( "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" @@ -29,7 +29,7 @@ var _ = Describe("recorder.Provider", func() { makeBroadcaster := func() (record.EventBroadcaster, bool) { return record.NewBroadcaster(), true } Describe("NewProvider", func() { It("should return a provider instance and a nil error.", func() { - provider, err := recorder.NewProvider(cfg, scheme.Scheme, logr.DiscardLogger{}, makeBroadcaster) + provider, err := recorder.NewProvider(cfg, httpClient, scheme.Scheme, logr.Discard(), makeBroadcaster) Expect(provider).NotTo(BeNil()) Expect(err).NotTo(HaveOccurred()) }) @@ -38,14 +38,14 @@ var _ = Describe("recorder.Provider", func() { // Invalid the config cfg1 := *cfg cfg1.Host = "invalid host" - _, err := recorder.NewProvider(&cfg1, scheme.Scheme, logr.DiscardLogger{}, makeBroadcaster) - Expect(err).NotTo(BeNil()) + _, err := recorder.NewProvider(&cfg1, httpClient, scheme.Scheme, logr.Discard(), makeBroadcaster) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to init client")) }) }) Describe("GetEventRecorder", func() { It("should return a recorder instance.", func() { - provider, err := recorder.NewProvider(cfg, scheme.Scheme, logr.DiscardLogger{}, makeBroadcaster) + provider, err := recorder.NewProvider(cfg, httpClient, scheme.Scheme, logr.Discard(), makeBroadcaster) Expect(err).NotTo(HaveOccurred()) recorder := provider.GetEventRecorderFor("test") diff --git a/pkg/source/internal/eventsource.go b/pkg/internal/source/event_handler.go similarity index 60% rename from pkg/source/internal/eventsource.go rename to pkg/internal/source/event_handler.go index f0cfe212ed..7cc8c51555 100644 --- a/pkg/source/internal/eventsource.go +++ b/pkg/internal/source/event_handler.go @@ -17,6 +17,7 @@ limitations under the License. package internal import ( + "context" "fmt" "k8s.io/client-go/tools/cache" @@ -31,21 +32,41 @@ import ( var log = logf.RuntimeLog.WithName("source").WithName("EventHandler") -var _ cache.ResourceEventHandler = EventHandler{} +var _ cache.ResourceEventHandler = &EventHandler[client.Object, any]{} + +// NewEventHandler creates a new EventHandler. +func NewEventHandler[object client.Object, request comparable]( + ctx context.Context, + queue workqueue.TypedRateLimitingInterface[request], + handler handler.TypedEventHandler[object, request], + predicates []predicate.TypedPredicate[object]) *EventHandler[object, request] { + return &EventHandler[object, request]{ + ctx: ctx, + handler: handler, + queue: queue, + predicates: predicates, + } +} // EventHandler adapts a handler.EventHandler interface to a cache.ResourceEventHandler interface. -type EventHandler struct { - EventHandler handler.EventHandler - Queue workqueue.RateLimitingInterface - Predicates []predicate.Predicate +type EventHandler[object client.Object, request comparable] struct { + // ctx stores the context that created the event handler + // that is used to propagate cancellation signals to each handler function. + ctx context.Context + + handler handler.TypedEventHandler[object, request] + queue workqueue.TypedRateLimitingInterface[request] + predicates []predicate.TypedPredicate[object] } // OnAdd creates CreateEvent and calls Create on EventHandler. -func (e EventHandler) OnAdd(obj interface{}) { - c := event.CreateEvent{} +func (e *EventHandler[object, request]) OnAdd(obj interface{}, isInInitialList bool) { + c := event.TypedCreateEvent[object]{ + IsInInitialList: isInInitialList, + } // Pull Object out of the object - if o, ok := obj.(client.Object); ok { + if o, ok := obj.(object); ok { c.Object = o } else { log.Error(nil, "OnAdd missing Object", @@ -53,21 +74,23 @@ func (e EventHandler) OnAdd(obj interface{}) { return } - for _, p := range e.Predicates { + for _, p := range e.predicates { if !p.Create(c) { return } } // Invoke create handler - e.EventHandler.Create(c, e.Queue) + ctx, cancel := context.WithCancel(e.ctx) + defer cancel() + e.handler.Create(ctx, c, e.queue) } // OnUpdate creates UpdateEvent and calls Update on EventHandler. -func (e EventHandler) OnUpdate(oldObj, newObj interface{}) { - u := event.UpdateEvent{} +func (e *EventHandler[object, request]) OnUpdate(oldObj, newObj interface{}) { + u := event.TypedUpdateEvent[object]{} - if o, ok := oldObj.(client.Object); ok { + if o, ok := oldObj.(object); ok { u.ObjectOld = o } else { log.Error(nil, "OnUpdate missing ObjectOld", @@ -76,7 +99,7 @@ func (e EventHandler) OnUpdate(oldObj, newObj interface{}) { } // Pull Object out of the object - if o, ok := newObj.(client.Object); ok { + if o, ok := newObj.(object); ok { u.ObjectNew = o } else { log.Error(nil, "OnUpdate missing ObjectNew", @@ -84,19 +107,21 @@ func (e EventHandler) OnUpdate(oldObj, newObj interface{}) { return } - for _, p := range e.Predicates { + for _, p := range e.predicates { if !p.Update(u) { return } } // Invoke update handler - e.EventHandler.Update(u, e.Queue) + ctx, cancel := context.WithCancel(e.ctx) + defer cancel() + e.handler.Update(ctx, u, e.queue) } // OnDelete creates DeleteEvent and calls Delete on EventHandler. -func (e EventHandler) OnDelete(obj interface{}) { - d := event.DeleteEvent{} +func (e *EventHandler[object, request]) OnDelete(obj interface{}) { + d := event.TypedDeleteEvent[object]{} // Deal with tombstone events by pulling the object out. Tombstone events wrap the object in a // DeleteFinalStateUnknown struct, so the object needs to be pulled out. @@ -114,12 +139,15 @@ func (e EventHandler) OnDelete(obj interface{}) { return } + // Set DeleteStateUnknown to true + d.DeleteStateUnknown = true + // Set obj to the tombstone obj obj = tombstone.Obj } // Pull Object out of the object - if o, ok := obj.(client.Object); ok { + if o, ok := obj.(object); ok { d.Object = o } else { log.Error(nil, "OnDelete missing Object", @@ -127,12 +155,14 @@ func (e EventHandler) OnDelete(obj interface{}) { return } - for _, p := range e.Predicates { + for _, p := range e.predicates { if !p.Delete(d) { return } } // Invoke delete handler - e.EventHandler.Delete(d, e.Queue) + ctx, cancel := context.WithCancel(e.ctx) + defer cancel() + e.handler.Delete(ctx, d, e.queue) } diff --git a/pkg/source/internal/internal_suite_test.go b/pkg/internal/source/internal_suite_test.go similarity index 78% rename from pkg/source/internal/internal_suite_test.go rename to pkg/internal/source/internal_suite_test.go index 21dd5ee6b4..eeee8b22cd 100644 --- a/pkg/source/internal/internal_suite_test.go +++ b/pkg/internal/source/internal_suite_test.go @@ -19,17 +19,15 @@ package internal_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestInternal(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Source Internal Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Source Internal Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/source/internal/internal_test.go b/pkg/internal/source/internal_test.go similarity index 60% rename from pkg/source/internal/internal_test.go rename to pkg/internal/source/internal_test.go index 9b96c6d46d..73eb1a1d28 100644 --- a/pkg/source/internal/internal_test.go +++ b/pkg/internal/source/internal_test.go @@ -17,13 +17,17 @@ limitations under the License. package internal_test import ( - . "github.com/onsi/ginkgo" + "context" + + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/source/internal" + internal "sigs.k8s.io/controller-runtime/pkg/internal/source" + "sigs.k8s.io/controller-runtime/pkg/reconcile" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,48 +38,44 @@ import ( ) var _ = Describe("Internal", func() { - - var instance internal.EventHandler + var instance *internal.EventHandler[client.Object, reconcile.Request] var funcs, setfuncs *handler.Funcs var set bool - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { funcs = &handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect CreateEvent to be called.") }, - DeleteFunc: func(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect DeleteEvent to be called.") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect UpdateEvent to be called.") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect GenericEvent to be called.") }, } setfuncs = &handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, - DeleteFunc: func(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, } - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: funcs, - } + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, funcs, nil) }) Describe("EventHandler", func() { @@ -91,120 +91,107 @@ var _ = Describe("Internal", func() { newPod.Labels = map[string]string{"foo": "bar"} }) - It("should create a CreateEvent", func() { - funcs.CreateFunc = func(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + It("should create a CreateEvent", func(ctx SpecContext) { + funcs.CreateFunc = func(ctx context.Context, evt event.CreateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) Expect(evt.Object).To(Equal(pod)) } - instance.OnAdd(pod) + instance.OnAdd(pod, false) }) - It("should used Predicates to filter CreateEvents", func() { - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: setfuncs, - } - - set = false - instance.Predicates = []predicate.Predicate{ + It("should used Predicates to filter CreateEvents", func(ctx SpecContext) { + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, - } - instance.OnAdd(pod) + }) + set = false + instance.OnAdd(pod, false) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } - instance.OnAdd(pod) + }) + instance.OnAdd(pod, false) Expect(set).To(BeTrue()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, - } - instance.OnAdd(pod) + }) + instance.OnAdd(pod, false) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } - instance.OnAdd(pod) + }) + instance.OnAdd(pod, false) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } - instance.OnAdd(pod) + }) + instance.OnAdd(pod, false) Expect(set).To(BeTrue()) }) It("should not call Create EventHandler if the object is not a runtime.Object", func() { - instance.OnAdd(&metav1.ObjectMeta{}) + instance.OnAdd(&metav1.ObjectMeta{}, false) }) It("should not call Create EventHandler if the object does not have metadata", func() { - instance.OnAdd(FooRuntimeObject{}) + instance.OnAdd(FooRuntimeObject{}, false) }) - It("should create an UpdateEvent", func() { - funcs.UpdateFunc = func(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + It("should create an UpdateEvent", func(ctx SpecContext) { + funcs.UpdateFunc = func(ctx context.Context, evt event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) - Expect(evt.ObjectOld).To(Equal(pod)) Expect(evt.ObjectNew).To(Equal(newPod)) } instance.OnUpdate(pod, newPod) }) - It("should used Predicates to filter UpdateEvents", func() { - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: setfuncs, - } - + It("should used Predicates to filter UpdateEvents", func(ctx SpecContext) { set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(updateEvent event.UpdateEvent) bool { return false }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return true }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeTrue()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return true }}, predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return false }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return false }}, predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return true }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeTrue()) }) @@ -220,56 +207,49 @@ var _ = Describe("Internal", func() { }) It("should create a DeleteEvent", func() { - funcs.DeleteFunc = func(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) - Expect(evt.Object).To(Equal(pod)) } instance.OnDelete(pod) }) - It("should used Predicates to filter DeleteEvents", func() { - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: setfuncs, - } - + It("should used Predicates to filter DeleteEvents", func(ctx SpecContext) { set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return false }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeTrue()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return false }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return false }}, predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeTrue()) }) @@ -283,14 +263,13 @@ var _ = Describe("Internal", func() { }) It("should create a DeleteEvent from a tombstone", func() { - tombstone := cache.DeletedFinalStateUnknown{ Obj: pod, } - funcs.DeleteFunc = func(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) Expect(evt.Object).To(Equal(pod)) + Expect(evt.DeleteStateUnknown).Should(BeTrue()) } instance.OnDelete(tombstone) @@ -301,11 +280,20 @@ var _ = Describe("Internal", func() { instance.OnDelete(tombstone) }) It("should ignore objects without meta", func() { - instance.OnAdd(Foo{}) + instance.OnAdd(Foo{}, false) instance.OnUpdate(Foo{}, Foo{}) instance.OnDelete(Foo{}) }) }) + + Describe("Kind", func() { + It("should return kind source type", func() { + kind := internal.Kind[*corev1.Pod, reconcile.Request]{ + Type: &corev1.Pod{}, + } + Expect(kind.String()).Should(Equal("kind source: *v1.Pod")) + }) + }) }) type Foo struct{} diff --git a/pkg/internal/source/kind.go b/pkg/internal/source/kind.go new file mode 100644 index 0000000000..2854244523 --- /dev/null +++ b/pkg/internal/source/kind.go @@ -0,0 +1,143 @@ +package internal + +import ( + "context" + "errors" + "fmt" + "reflect" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + toolscache "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" + + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +var logKind = logf.RuntimeLog.WithName("source").WithName("Kind") + +// Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create). +type Kind[object client.Object, request comparable] struct { + // Type is the type of object to watch. e.g. &v1.Pod{} + Type object + + // Cache used to watch APIs + Cache cache.Cache + + Handler handler.TypedEventHandler[object, request] + + Predicates []predicate.TypedPredicate[object] + + // startedErr may contain an error if one was encountered during startup. If its closed and does not + // contain an error, startup and syncing finished. + startedErr chan error + startCancel func() +} + +// Start is internal and should be called only by the Controller to register an EventHandler with the Informer +// to enqueue reconcile.Requests. +func (ks *Kind[object, request]) Start(ctx context.Context, queue workqueue.TypedRateLimitingInterface[request]) error { + if isNil(ks.Type) { + return fmt.Errorf("must create Kind with a non-nil object") + } + if isNil(ks.Cache) { + return fmt.Errorf("must create Kind with a non-nil cache") + } + if isNil(ks.Handler) { + return errors.New("must create Kind with non-nil handler") + } + + // cache.GetInformer will block until its context is cancelled if the cache was already started and it can not + // sync that informer (most commonly due to RBAC issues). + ctx, ks.startCancel = context.WithCancel(ctx) + ks.startedErr = make(chan error, 1) // Buffer chan to not leak goroutines if WaitForSync isn't called + go func() { + var ( + i cache.Informer + lastErr error + ) + + // Tries to get an informer until it returns true, + // an error or the specified context is cancelled or expired. + if err := wait.PollUntilContextCancel(ctx, 10*time.Second, true, func(ctx context.Context) (bool, error) { + // Lookup the Informer from the Cache and add an EventHandler which populates the Queue + i, lastErr = ks.Cache.GetInformer(ctx, ks.Type) + if lastErr != nil { + kindMatchErr := &meta.NoKindMatchError{} + switch { + case errors.As(lastErr, &kindMatchErr): + logKind.Error(lastErr, "if kind is a CRD, it should be installed before calling Start", + "kind", kindMatchErr.GroupKind) + case runtime.IsNotRegisteredError(lastErr): + logKind.Error(lastErr, "kind must be registered to the Scheme") + default: + logKind.Error(lastErr, "failed to get informer from cache") + } + return false, nil // Retry. + } + return true, nil + }); err != nil { + if lastErr != nil { + ks.startedErr <- fmt.Errorf("failed to get informer from cache: %w", lastErr) + return + } + ks.startedErr <- err + return + } + + _, err := i.AddEventHandlerWithOptions(NewEventHandler(ctx, queue, ks.Handler, ks.Predicates), toolscache.HandlerOptions{ + Logger: &logKind, + }) + if err != nil { + ks.startedErr <- err + return + } + if !ks.Cache.WaitForCacheSync(ctx) { + // Would be great to return something more informative here + ks.startedErr <- errors.New("cache did not sync") + } + close(ks.startedErr) + }() + + return nil +} + +func (ks *Kind[object, request]) String() string { + if !isNil(ks.Type) { + return fmt.Sprintf("kind source: %T", ks.Type) + } + return "kind source: unknown type" +} + +// WaitForSync implements SyncingSource to allow controllers to wait with starting +// workers until the cache is synced. +func (ks *Kind[object, request]) WaitForSync(ctx context.Context) error { + select { + case err := <-ks.startedErr: + return err + case <-ctx.Done(): + ks.startCancel() + if errors.Is(ctx.Err(), context.Canceled) { + return nil + } + return fmt.Errorf("timed out waiting for cache to be synced for Kind %T", ks.Type) + } +} + +func isNil(arg any) bool { + if v := reflect.ValueOf(arg); !v.IsValid() || ((v.Kind() == reflect.Ptr || + v.Kind() == reflect.Interface || + v.Kind() == reflect.Slice || + v.Kind() == reflect.Map || + v.Kind() == reflect.Chan || + v.Kind() == reflect.Func) && v.IsNil()) { + return true + } + return false +} diff --git a/pkg/internal/syncs/syncs.go b/pkg/internal/syncs/syncs.go new file mode 100644 index 0000000000..c78a30377a --- /dev/null +++ b/pkg/internal/syncs/syncs.go @@ -0,0 +1,38 @@ +package syncs + +import ( + "context" + "reflect" + "sync" +) + +// MergeChans returns a channel that is closed when any of the input channels are signaled. +// The caller must call the returned CancelFunc to ensure no resources are leaked. +func MergeChans[T any](chans ...<-chan T) (<-chan T, context.CancelFunc) { + var once sync.Once + out := make(chan T) + cancel := make(chan T) + cancelFunc := func() { + once.Do(func() { + close(cancel) + }) + <-out + } + cases := make([]reflect.SelectCase, len(chans)+1) + for i := range chans { + cases[i] = reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(chans[i]), + } + } + cases[len(cases)-1] = reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(cancel), + } + go func() { + defer close(out) + _, _, _ = reflect.Select(cases) + }() + + return out, cancelFunc +} diff --git a/pkg/internal/syncs/syncs_test.go b/pkg/internal/syncs/syncs_test.go new file mode 100644 index 0000000000..7bf7d598a0 --- /dev/null +++ b/pkg/internal/syncs/syncs_test.go @@ -0,0 +1,107 @@ +package syncs + +import ( + "testing" + "time" + + // This appears to be needed so that the prow test runner won't fail. + _ "github.com/onsi/ginkgo/v2" + _ "github.com/onsi/gomega" +) + +func TestMergeChans(t *testing.T) { + tests := []struct { + name string + count int + signal int + }{ + { + name: "single channel, close 0", + count: 1, + signal: 0, + }, + { + name: "double channel, close 0", + count: 2, + signal: 0, + }, + { + name: "five channel, close 0", + count: 5, + signal: 0, + }, + { + name: "five channel, close 1", + count: 5, + signal: 1, + }, + { + name: "five channel, close 2", + count: 5, + signal: 2, + }, + { + name: "five channel, close 3", + count: 5, + signal: 3, + }, + { + name: "five channel, close 4", + count: 5, + signal: 4, + }, + { + name: "single channel, cancel", + count: 1, + signal: -1, + }, + { + name: "double channel, cancel", + count: 2, + signal: -1, + }, + { + name: "five channel, cancel", + count: 5, + signal: -1, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if callAndClose(test.count, test.signal, 1) { + t.Error("timeout before merged channel closed") + } + }) + } +} + +func callAndClose(numChans, signalChan, timeoutSeconds int) bool { + chans := make([]chan struct{}, numChans) + readOnlyChans := make([]<-chan struct{}, numChans) + for i := range chans { + chans[i] = make(chan struct{}) + readOnlyChans[i] = chans[i] + } + defer func() { + for i := range chans { + close(chans[i]) + } + }() + + merged, cancel := MergeChans(readOnlyChans...) + defer cancel() + + timer := time.NewTimer(time.Duration(timeoutSeconds) * time.Second) + + if signalChan >= 0 { + chans[signalChan] <- struct{}{} + } else { + cancel() + } + select { + case <-merged: + return false + case <-timer.C: + return true + } +} diff --git a/pkg/internal/testing/OWNERS b/pkg/internal/testing/OWNERS deleted file mode 100644 index 25fda2ebac..0000000000 --- a/pkg/internal/testing/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/devel/owners.md - -approvers: - - testing-integration-approvers diff --git a/pkg/internal/testing/addr/addr_suite_test.go b/pkg/internal/testing/addr/addr_suite_test.go index b18c62def9..3869bb0207 100644 --- a/pkg/internal/testing/addr/addr_suite_test.go +++ b/pkg/internal/testing/addr/addr_suite_test.go @@ -19,15 +19,12 @@ package addr_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestAddr(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "Addr Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Addr Suite") } diff --git a/pkg/internal/testing/addr/manager.go b/pkg/internal/testing/addr/manager.go index caaafa2627..ffa33a8861 100644 --- a/pkg/internal/testing/addr/manager.go +++ b/pkg/internal/testing/addr/manager.go @@ -43,11 +43,17 @@ var ( func init() { baseDir, err := os.UserCacheDir() + if err == nil { + cacheDir = filepath.Join(baseDir, "kubebuilder-envtest") + err = os.MkdirAll(cacheDir, 0o750) + } if err != nil { + // Either we didn't get a cache directory, or we can't use it baseDir = os.TempDir() + cacheDir = filepath.Join(baseDir, "kubebuilder-envtest") + err = os.MkdirAll(cacheDir, 0o750) } - cacheDir = filepath.Join(baseDir, "kubebuilder-envtest") - if err := os.MkdirAll(cacheDir, 0750); err != nil { + if err != nil { panic(err) } } @@ -65,10 +71,20 @@ func (c *portCache) add(port int) (bool, error) { } info, err := d.Info() if err != nil { + // No-op if file no longer exists; may have been deleted by another + // process/thread trying to allocate ports. + if errors.Is(err, fs.ErrNotExist) { + return nil + } return err } if time.Since(info.ModTime()) > portReserveTime { if err := os.Remove(filepath.Join(cacheDir, path)); err != nil { + // No-op if file no longer exists; may have been deleted by another + // process/thread trying to allocate ports. + if os.IsNotExist(err) { + return nil + } return err } } diff --git a/pkg/internal/testing/addr/manager_test.go b/pkg/internal/testing/addr/manager_test.go index cf95c36115..065e847dc5 100644 --- a/pkg/internal/testing/addr/manager_test.go +++ b/pkg/internal/testing/addr/manager_test.go @@ -20,7 +20,7 @@ import ( "net" "strconv" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" diff --git a/pkg/internal/testing/certs/certs_suite_test.go b/pkg/internal/testing/certs/certs_suite_test.go index 5b63fc4f55..3b3008c294 100644 --- a/pkg/internal/testing/certs/certs_suite_test.go +++ b/pkg/internal/testing/certs/certs_suite_test.go @@ -19,15 +19,12 @@ package certs_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestInternal(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "TinyCA (Internal Certs) Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "TinyCA (Internal Certs) Suite") } diff --git a/pkg/internal/testing/certs/tinyca.go b/pkg/internal/testing/certs/tinyca.go index 55b044c5b3..b4188237e6 100644 --- a/pkg/internal/testing/certs/tinyca.go +++ b/pkg/internal/testing/certs/tinyca.go @@ -64,7 +64,7 @@ func (k CertPair) AsBytes() (cert []byte, key []byte, err error) { rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key) if err != nil { - return nil, nil, fmt.Errorf("unable to encode private key: %v", err) + return nil, nil, fmt.Errorf("unable to encode private key: %w", err) } key = pem.EncodeToMemory(&pem.Block{ @@ -95,12 +95,12 @@ func newPrivateKey() (crypto.Signer, error) { func NewTinyCA() (*TinyCA, error) { caPrivateKey, err := newPrivateKey() if err != nil { - return nil, fmt.Errorf("unable to generate private key for CA: %v", err) + return nil, fmt.Errorf("unable to generate private key for CA: %w", err) } caCfg := certutil.Config{CommonName: "envtest-environment", Organization: []string{"envtest"}} caCert, err := certutil.NewSelfSignedCACert(caCfg, caPrivateKey) if err != nil { - return nil, fmt.Errorf("unable to generate certificate for CA: %v", err) + return nil, fmt.Errorf("unable to generate certificate for CA: %w", err) } return &TinyCA{ @@ -115,7 +115,7 @@ func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) { key, err := newPrivateKey() if err != nil { - return CertPair{}, fmt.Errorf("unable to create private key: %v", err) + return CertPair{}, fmt.Errorf("unable to create private key: %w", err) } serial := new(big.Int).Set(c.nextSerial) @@ -140,12 +140,12 @@ func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) { certRaw, err := x509.CreateCertificate(crand.Reader, &template, c.CA.Cert, key.Public(), c.CA.Key) if err != nil { - return CertPair{}, fmt.Errorf("unable to create certificate: %v", err) + return CertPair{}, fmt.Errorf("unable to create certificate: %w", err) } cert, err := x509.ParseCertificate(certRaw) if err != nil { - return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %v", err) + return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %w", err) } return CertPair{ diff --git a/pkg/internal/testing/certs/tinyca_test.go b/pkg/internal/testing/certs/tinyca_test.go index e3f2513210..5d84de56fb 100644 --- a/pkg/internal/testing/certs/tinyca_test.go +++ b/pkg/internal/testing/certs/tinyca_test.go @@ -24,7 +24,7 @@ import ( "sort" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" @@ -47,8 +47,8 @@ var _ = Describe("TinyCA", func() { }) It("should be usable for signing & verifying", func() { - Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageCertSign).NotTo(Equal(0), "should be usable for cert signing") - Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(Equal(0), "should be usable for signature verifying") + Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageCertSign).NotTo(BeEquivalentTo(0), "should be usable for cert signing") + Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") }) }) @@ -141,8 +141,8 @@ var _ = Describe("TinyCA", func() { cert, err := ca.NewServingCert() Expect(err).NotTo(HaveOccurred(), "should be able to generate a serving cert") - Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(Equal(0), "should be usable for key enciphering") - Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(Equal(0), "should be usable for signature verifying") + Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(BeEquivalentTo(0), "should be usable for key enciphering") + Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") Expect(cert.Cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageServerAuth), "should be usable for server auth") }) @@ -172,8 +172,8 @@ var _ = Describe("TinyCA", func() { }) It("should be usable for client auth, verifying, and enciphering", func() { - Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(Equal(0), "should be usable for key enciphering") - Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(Equal(0), "should be usable for signature verifying") + Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(BeEquivalentTo(0), "should be usable for key enciphering") + Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") Expect(cert.Cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageClientAuth), "should be usable for client auth") }) diff --git a/pkg/internal/testing/controlplane/apiserver.go b/pkg/internal/testing/controlplane/apiserver.go index ece5efd651..aadb69e84f 100644 --- a/pkg/internal/testing/controlplane/apiserver.go +++ b/pkg/internal/testing/controlplane/apiserver.go @@ -19,7 +19,6 @@ package controlplane import ( "fmt" "io" - "io/ioutil" "net/url" "os" "path/filepath" @@ -375,7 +374,12 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - servingCerts, err := ca.NewServingCert() + servingAddresses := []string{"localhost"} + if s.SecureServing.ListenAddr.Address != "" { + servingAddresses = append(servingAddresses, s.SecureServing.ListenAddr.Address) + } + + servingCerts, err := ca.NewServingCert(servingAddresses...) if err != nil { return err } @@ -385,10 +389,10 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0o640); err != nil { return err } - if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0o640); err != nil { return err } @@ -405,23 +409,26 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - if err := ioutil.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0o640); err != nil { return err } - return ioutil.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec + return os.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0o640) } // Stop stops this process gracefully, waits for its termination, and cleans up // the CertDir if necessary. func (s *APIServer) Stop() error { - if s.processState.DirNeedsCleaning { - s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart - } if s.processState != nil { + if s.processState.DirNeedsCleaning { + s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart + } if err := s.processState.Stop(); err != nil { return err } } + if s.Authn == nil { + return nil + } return s.Authn.Stop() } diff --git a/pkg/internal/testing/controlplane/apiserver_test.go b/pkg/internal/testing/controlplane/apiserver_test.go index b857220203..0811e9fb59 100644 --- a/pkg/internal/testing/controlplane/apiserver_test.go +++ b/pkg/internal/testing/controlplane/apiserver_test.go @@ -17,10 +17,15 @@ limitations under the License. package controlplane_test import ( + "crypto/x509" + "encoding/pem" "errors" + "net" "net/url" + "os" + "path" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" @@ -128,7 +133,7 @@ var _ = Describe("APIServer", func() { Context("when SecureServing is not set", func() { It("should be defaulted with a random port", func() { - Expect(server.Port).NotTo(Equal(0)) + Expect(server.Port).NotTo(BeEquivalentTo(0)) }) }) }) @@ -191,6 +196,66 @@ var _ = Describe("APIServer", func() { }) }) + // These tests assume that 'localhost' resolves to 127.0.0.1. It can resolve + // to other addresses as well (e.g. ::1 on IPv6), but it must always resolve + // to 127.0.0.1. + Describe(("generated certificates"), func() { + getCertificate := func() *x509.Certificate { + // Read the cert file + certFile := path.Join(server.CertDir, "apiserver.crt") + certBytes, err := os.ReadFile(certFile) + Expect(err).NotTo(HaveOccurred(), "should be able to read the cert file") + + // Decode and parse it + block, remainder := pem.Decode(certBytes) + Expect(block).NotTo(BeNil(), "should be able to decode the cert file") + Expect(remainder).To(BeEmpty(), "should not have any extra data in the cert file") + Expect(block.Type).To(Equal("CERTIFICATE"), "should be a certificate block") + + cert, err := x509.ParseCertificate(block.Bytes) + Expect(err).NotTo(HaveOccurred(), "should be able to parse the cert file") + + return cert + } + + Context("when SecureServing are not set", func() { + It("should have localhost/127.0.0.1 in the certificate altnames", func() { + cert := getCertificate() + + Expect(cert.Subject.CommonName).To(Equal("localhost")) + Expect(cert.DNSNames).To(ConsistOf("localhost")) + expectedIPAddresses := []net.IP{ + net.ParseIP("127.0.0.1").To4(), + net.ParseIP(server.SecureServing.ListenAddr.Address).To4(), + } + Expect(cert.IPAddresses).To(ContainElements(expectedIPAddresses)) + }) + }) + + Context("when SecureServing host & port are set", func() { + BeforeEach(func() { + server.SecureServing = SecureServing{ + ListenAddr: process.ListenAddr{ + Address: "1.2.3.4", + Port: "5678", + }, + } + }) + + It("should have the host in the certificate altnames", func() { + cert := getCertificate() + + Expect(cert.Subject.CommonName).To(Equal("localhost")) + Expect(cert.DNSNames).To(ConsistOf("localhost")) + expectedIPAddresses := []net.IP{ + net.ParseIP("127.0.0.1").To4(), + net.ParseIP(server.SecureServing.ListenAddr.Address).To4(), + } + Expect(cert.IPAddresses).To(ContainElements(expectedIPAddresses)) + }) + }) + }) + Describe("setting up auth", func() { var auth *fakeAuthn BeforeEach(func() { diff --git a/pkg/internal/testing/controlplane/auth.go b/pkg/internal/testing/controlplane/auth.go index b2cd4e5e04..b44035ebf2 100644 --- a/pkg/internal/testing/controlplane/auth.go +++ b/pkg/internal/testing/controlplane/auth.go @@ -18,7 +18,7 @@ package controlplane import ( "fmt" - "io/ioutil" + "os" "path/filepath" "k8s.io/client-go/rest" @@ -128,7 +128,7 @@ func (c *CertAuthn) Start() error { return fmt.Errorf("start called before configure") } caCrt := c.ca.CA.CertBytes() - if err := ioutil.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { return fmt.Errorf("unable to save the client certificate CA to %s: %w", c.caCrtPath(), err) } diff --git a/pkg/internal/testing/controlplane/auth_test.go b/pkg/internal/testing/controlplane/auth_test.go index 3acbc3d3c4..9891c6f2e2 100644 --- a/pkg/internal/testing/controlplane/auth_test.go +++ b/pkg/internal/testing/controlplane/auth_test.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" kcert "k8s.io/client-go/util/cert" diff --git a/pkg/internal/testing/controlplane/controlplane_suite_test.go b/pkg/internal/testing/controlplane/controlplane_suite_test.go index 067b0c40ce..9ac69047f0 100644 --- a/pkg/internal/testing/controlplane/controlplane_suite_test.go +++ b/pkg/internal/testing/controlplane/controlplane_suite_test.go @@ -19,15 +19,12 @@ package controlplane_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestIntegration(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "Control Plane Standup Unit Tests" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Control Plane Standup Unit Tests") } diff --git a/pkg/internal/testing/controlplane/etcd.go b/pkg/internal/testing/controlplane/etcd.go index c6d6251731..98ffe3ac5e 100644 --- a/pkg/internal/testing/controlplane/etcd.go +++ b/pkg/internal/testing/controlplane/etcd.go @@ -91,7 +91,7 @@ type Etcd struct { } // Start starts the etcd, waits for it to come up, and returns an error, if one -// occoured. +// occurred. func (e *Etcd) Start() error { if err := e.setProcessState(); err != nil { return err @@ -159,6 +159,10 @@ func (e *Etcd) setProcessState() error { // Stop stops this process gracefully, waits for its termination, and cleans up // the DataDir if necessary. func (e *Etcd) Stop() error { + if e.processState == nil { + return nil + } + if e.processState.DirNeedsCleaning { e.DataDir = "" // reset the directory if it was randomly allocated, so that we can safely restart } diff --git a/pkg/internal/testing/controlplane/etcd_test.go b/pkg/internal/testing/controlplane/etcd_test.go index e9a1f7a181..7c7c7561ff 100644 --- a/pkg/internal/testing/controlplane/etcd_test.go +++ b/pkg/internal/testing/controlplane/etcd_test.go @@ -17,7 +17,7 @@ limitations under the License. package controlplane_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/pkg/internal/testing/controlplane" diff --git a/pkg/internal/testing/controlplane/kubectl.go b/pkg/internal/testing/controlplane/kubectl.go index a27b7a0ff8..a41bb77c4d 100644 --- a/pkg/internal/testing/controlplane/kubectl.go +++ b/pkg/internal/testing/controlplane/kubectl.go @@ -112,6 +112,7 @@ func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) { cmd := exec.Command(k.Path, allArgs...) cmd.Stdout = stdoutBuffer cmd.Stderr = stderrBuffer + cmd.SysProcAttr = process.GetSysProcAttr() err = cmd.Run() diff --git a/pkg/internal/testing/controlplane/kubectl_test.go b/pkg/internal/testing/controlplane/kubectl_test.go index d7105b29f2..5484bc31a1 100644 --- a/pkg/internal/testing/controlplane/kubectl_test.go +++ b/pkg/internal/testing/controlplane/kubectl_test.go @@ -17,9 +17,9 @@ limitations under the License. package controlplane_test import ( - "io/ioutil" + "io" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "k8s.io/client-go/rest" @@ -36,7 +36,7 @@ var _ = Describe("Kubectl", func() { stdout, stderr, err := k.Run(args...) Expect(err).NotTo(HaveOccurred()) Expect(stdout).To(ContainSubstring("something")) - bytes, err := ioutil.ReadAll(stderr) + bytes, err := io.ReadAll(stderr) Expect(err).NotTo(HaveOccurred()) Expect(bytes).To(BeEmpty()) }) diff --git a/pkg/internal/testing/controlplane/plane.go b/pkg/internal/testing/controlplane/plane.go index 36fd3c6306..456183a7a3 100644 --- a/pkg/internal/testing/controlplane/plane.go +++ b/pkg/internal/testing/controlplane/plane.go @@ -47,13 +47,18 @@ type ControlPlane struct { } // Start will start your control plane processes. To stop them, call Stop(). -func (f *ControlPlane) Start() error { +func (f *ControlPlane) Start() (retErr error) { if f.Etcd == nil { f.Etcd = &Etcd{} } if err := f.Etcd.Start(); err != nil { return err } + defer func() { + if retErr != nil { + _ = f.Etcd.Stop() + } + }() if f.APIServer == nil { f.APIServer = &APIServer{} @@ -62,6 +67,11 @@ func (f *ControlPlane) Start() error { if err := f.APIServer.Start(); err != nil { return err } + defer func() { + if retErr != nil { + _ = f.APIServer.Stop() + } + }() // provision the default user -- can be removed when the related // methods are removed. The default user has admin permissions to @@ -88,6 +98,7 @@ func (f *ControlPlane) Stop() error { errList = append(errList, err) } } + if f.Etcd != nil { if err := f.Etcd.Stop(); err != nil { errList = append(errList, err) diff --git a/pkg/internal/testing/controlplane/plane_test.go b/pkg/internal/testing/controlplane/plane_test.go index 714e76e8a4..a228e5a51c 100644 --- a/pkg/internal/testing/controlplane/plane_test.go +++ b/pkg/internal/testing/controlplane/plane_test.go @@ -17,9 +17,7 @@ limitations under the License. package controlplane_test import ( - "context" - - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" kauthn "k8s.io/api/authorization/v1" @@ -69,7 +67,7 @@ var _ = Describe("Control Plane", func() { Expect(plane.Stop()).To(Succeed()) }) - It("should provision a working legacy user and legacy kubectl", func() { + It("should provision a working legacy user and legacy kubectl", func(ctx SpecContext) { By("grabbing the legacy kubectl") Expect(plane.KubeCtl()).NotTo(BeNil()) @@ -89,7 +87,7 @@ var _ = Describe("Control Plane", func() { }, }, } - Expect(cl.Create(context.Background(), sar)).To(Succeed(), "should be able to make a Self-SAR") + Expect(cl.Create(ctx, sar)).To(Succeed(), "should be able to make a Self-SAR") Expect(sar.Status.Allowed).To(BeTrue(), "admin user should be able to do everything") }) diff --git a/pkg/internal/testing/process/arguments.go b/pkg/internal/testing/process/arguments.go index 6c2c91e144..391eec1fac 100644 --- a/pkg/internal/testing/process/arguments.go +++ b/pkg/internal/testing/process/arguments.go @@ -93,12 +93,12 @@ type TemplateDefaults struct { // TemplateAndArguments joins structured arguments and non-structured arguments, preserving existing // behavior. Namely: // -// 1. if templ has len > 0, it will be rendered against data -// 2. the rendered template values that look like `--foo=bar` will be split -// and appended to args, the rest will be kept around -// 3. the given args will be rendered as string form. If a template is given, -// no defaults will be used, otherwise defaults will be used -// 4. a result of [args..., rest...] will be returned +// 1. if templ has len > 0, it will be rendered against data +// 2. the rendered template values that look like `--foo=bar` will be split +// and appended to args, the rest will be kept around +// 3. the given args will be rendered as string form. If a template is given, +// no defaults will be used, otherwise defaults will be used +// 4. a result of [args..., rest...] will be returned // // It returns the resulting rendered arguments, plus the arguments that were // not transferred to `args` during rendering. @@ -215,9 +215,9 @@ var ( // for passing to exec.Command and friends, making use of the given defaults // as indicated for each particular argument. // -// - Any flag in defaults that's not in Arguments will be present in the output -// - Any flag that's present in Arguments will be passed the corresponding -// defaults to do with as it will (ignore, append-to, suppress, etc). +// - Any flag in defaults that's not in Arguments will be present in the output +// - Any flag that's present in Arguments will be passed the corresponding +// defaults to do with as it will (ignore, append-to, suppress, etc). func (a *Arguments) AsStrings(defaults map[string][]string) []string { // sort for deterministic ordering keysInOrder := make([]string, 0, len(defaults)+len(a.values)) @@ -323,9 +323,9 @@ func (a *Arguments) SetRaw(key string, val Arg) *Arguments { // used in conjunction with SetRaw. For example, to set `--some-flag` to the // API server's CertDir, you could do: // -// server.Configure().SetRaw("--some-flag", FuncArg(func(defaults []string) []string { -// return []string{server.CertDir} -// })) +// server.Configure().SetRaw("--some-flag", FuncArg(func(defaults []string) []string { +// return []string{server.CertDir} +// })) // // FuncArg ignores Appends; if you need to support appending values too, consider implementing // Arg directly. diff --git a/pkg/internal/testing/process/arguments_test.go b/pkg/internal/testing/process/arguments_test.go index 2534eb8446..b513cbdf86 100644 --- a/pkg/internal/testing/process/arguments_test.go +++ b/pkg/internal/testing/process/arguments_test.go @@ -20,7 +20,7 @@ import ( "net/url" "strings" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/pkg/internal/testing/process" @@ -225,12 +225,6 @@ var _ = Describe("Arguments Templates", func() { }) }) -type plainDefaults map[string][]string - -func (d plainDefaults) DefaultArgs() map[string][]string { - return d -} - var _ = Describe("Arguments", func() { Context("when appending", func() { It("should copy from defaults when appending for the first time", func() { diff --git a/pkg/internal/testing/process/bin_path_finder_test.go b/pkg/internal/testing/process/bin_path_finder_test.go index c933478811..425459e3aa 100644 --- a/pkg/internal/testing/process/bin_path_finder_test.go +++ b/pkg/internal/testing/process/bin_path_finder_test.go @@ -19,7 +19,7 @@ package process import ( "os" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -28,12 +28,12 @@ var _ = Describe("BinPathFinder", func() { BeforeEach(func() { prevAssetPath = os.Getenv(EnvAssetsPath) Expect(os.Unsetenv(EnvAssetsPath)).To(Succeed()) - Expect(os.Unsetenv(EnvAssetOverridePrefix + "_SOME_FAKE")) - Expect(os.Unsetenv(EnvAssetOverridePrefix + "OTHERFAKE")) + Expect(os.Unsetenv(EnvAssetOverridePrefix + "_SOME_FAKE")).To(Succeed()) + Expect(os.Unsetenv(EnvAssetOverridePrefix + "OTHERFAKE")).To(Succeed()) }) AfterEach(func() { if prevAssetPath != "" { - Expect(os.Setenv(EnvAssetsPath, prevAssetPath)) + Expect(os.Setenv(EnvAssetsPath, prevAssetPath)).To(Succeed()) } }) Context("when individual overrides are present", func() { diff --git a/pkg/ratelimiter/doc.go b/pkg/internal/testing/process/procattr_other.go similarity index 55% rename from pkg/ratelimiter/doc.go rename to pkg/internal/testing/process/procattr_other.go index a01d603fe5..df13b341a4 100644 --- a/pkg/ratelimiter/doc.go +++ b/pkg/internal/testing/process/procattr_other.go @@ -1,5 +1,8 @@ +//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !zos +// +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!zos + /* -Copyright 2020 The Kubernetes Authors. +Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,9 +17,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Package ratelimiter defines rate limiters used by Controllers to limit how frequently requests may be queued. +package process -Typical rate limiters that can be used are implemented in client-go's workqueue package. -*/ -package ratelimiter +import "syscall" + +// GetSysProcAttr returns the SysProcAttr to use for the process, +// for non-unix systems this returns nil. +func GetSysProcAttr() *syscall.SysProcAttr { + return nil +} diff --git a/pkg/envtest/ginkgo_test.go b/pkg/internal/testing/process/procattr_unix.go similarity index 50% rename from pkg/envtest/ginkgo_test.go rename to pkg/internal/testing/process/procattr_unix.go index fba031c954..83ad509af0 100644 --- a/pkg/envtest/ginkgo_test.go +++ b/pkg/internal/testing/process/procattr_unix.go @@ -1,5 +1,8 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris zos + /* -Copyright 2021 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,14 +17,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package envtest +package process import ( - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + "golang.org/x/sys/unix" ) -// NewlineReporter is Reporter that Prints a newline after the default Reporter output so that the results -// are correctly parsed by test automation. -// See issue https://github.com/jstemmer/go-junit-report/issues/31 -// It's re-exported here to avoid compatibility breakage/mass rewrites. -type NewlineReporter = printer.NewlineReporter +// GetSysProcAttr returns the SysProcAttr to use for the process, +// for unix systems this returns a SysProcAttr with Setpgid set to true, +// which inherits the parent's process group id. +func GetSysProcAttr() *unix.SysProcAttr { + return &unix.SysProcAttr{ + Setpgid: true, + } +} diff --git a/pkg/internal/testing/process/process.go b/pkg/internal/testing/process/process.go index 4675f9b947..0d541921e2 100644 --- a/pkg/internal/testing/process/process.go +++ b/pkg/internal/testing/process/process.go @@ -20,7 +20,6 @@ import ( "crypto/tls" "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -84,7 +83,7 @@ type State struct { DirNeedsCleaning bool Path string - // ready holds wether the process is currently in ready state (hit the ready condition) or not. + // ready holds whether the process is currently in ready state (hit the ready condition) or not. // It will be set to true on a successful `Start()` and set to false on a successful `Stop()` ready bool @@ -109,7 +108,7 @@ func (ps *State) Init(name string) error { } if ps.Dir == "" { - newDir, err := ioutil.TempDir("", "k8s_test_framework_") + newDir, err := os.MkdirTemp("", "k8s_test_framework_") if err != nil { return err } @@ -156,6 +155,7 @@ func (ps *State) Start(stdout, stderr io.Writer) (err error) { ps.Cmd = exec.Command(ps.Path, ps.Args...) ps.Cmd.Stdout = stdout ps.Cmd.Stderr = stderr + ps.Cmd.SysProcAttr = GetSysProcAttr() ready := make(chan bool) timedOut := time.After(ps.StartTimeout) @@ -185,16 +185,12 @@ func (ps *State) Start(stdout, stderr io.Writer) (err error) { ps.ready = true return nil case <-ps.waitDone: - if pollerStopCh != nil { - close(pollerStopCh) - } + close(pollerStopCh) return fmt.Errorf("timeout waiting for process %s to start successfully "+ "(it may have failed to start, or stopped unexpectedly before becoming ready)", path.Base(ps.Path)) case <-timedOut: - if pollerStopCh != nil { - close(pollerStopCh) - } + close(pollerStopCh) if ps.Cmd != nil { // intentionally ignore this -- we might've crashed, failed to start, etc ps.Cmd.Process.Signal(syscall.SIGTERM) //nolint:errcheck @@ -219,7 +215,7 @@ func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh // there's probably certs *somewhere*, // but it's fine to just skip validating // them for health checks during testing - InsecureSkipVerify: true, //nolint:gosec + InsecureSkipVerify: true, }, }, } @@ -270,6 +266,9 @@ func (ps *State) Stop() error { case <-ps.waitDone: break case <-timedOut: + if err := ps.Cmd.Process.Signal(syscall.SIGKILL); err != nil { + return fmt.Errorf("unable to kill process %s: %w", ps.Path, err) + } return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path)) } ps.ready = false diff --git a/pkg/internal/testing/process/process_suite_test.go b/pkg/internal/testing/process/process_suite_test.go index 4b9d7ab198..5a64e9d2f0 100644 --- a/pkg/internal/testing/process/process_suite_test.go +++ b/pkg/internal/testing/process/process_suite_test.go @@ -19,15 +19,12 @@ package process_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestInternal(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "Envtest Process Launcher Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Envtest Process Launcher Suite") } diff --git a/pkg/internal/testing/process/process_test.go b/pkg/internal/testing/process/process_test.go index 446d06707c..1d01e95b09 100644 --- a/pkg/internal/testing/process/process_test.go +++ b/pkg/internal/testing/process/process_test.go @@ -18,7 +18,6 @@ package process_test import ( "bytes" - "io/ioutil" "net" "net/http" "net/url" @@ -26,7 +25,7 @@ import ( "strconv" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" @@ -139,7 +138,7 @@ var _ = Describe("Start method", func() { echo 'i started' >&2 `, } - processState.StartTimeout = 1 * time.Second + processState.StartTimeout = 5 * time.Second Expect(processState.Start(stdout, stderr)).To(Succeed()) Eventually(processState.Exited).Should(BeTrue()) @@ -297,7 +296,7 @@ var _ = Describe("Stop method", func() { var err error Expect(processState.Start(nil, nil)).To(Succeed()) - processState.Dir, err = ioutil.TempDir("", "k8s_test_framework_") + processState.Dir, err = os.MkdirTemp("", "k8s_test_framework_") Expect(err).NotTo(HaveOccurred()) processState.DirNeedsCleaning = true processState.StopTimeout = 400 * time.Millisecond @@ -353,16 +352,7 @@ var _ = Describe("Init", func() { }) var simpleBashScript = []string{ - "-c", - ` - i=0 - while true - do - echo "loop $i" >&2 - let 'i += 1' - sleep 0.2 - done - `, + "-c", "tail -f /dev/null", } func getServerURL(server *ghttp.Server) url.URL { diff --git a/pkg/leaderelection/fake/leader_election.go b/pkg/leaderelection/fake/leader_election.go index 5a82cf43b8..ab816a19a7 100644 --- a/pkg/leaderelection/fake/leader_election.go +++ b/pkg/leaderelection/fake/leader_election.go @@ -19,7 +19,9 @@ package fake import ( "context" "encoding/json" + "errors" "os" + "sync/atomic" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,6 +32,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/recorder" ) +// ControllableResourceLockInterface is an interface that extends resourcelock.Interface to be +// controllable. +type ControllableResourceLockInterface interface { + resourcelock.Interface + + // BlockLeaderElection blocks the leader election process when called. It will not be unblocked + // until UnblockLeaderElection is called. + BlockLeaderElection() + + // UnblockLeaderElection unblocks the leader election. + UnblockLeaderElection() +} + // NewResourceLock creates a new ResourceLock for use in testing // leader election. func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { @@ -40,27 +55,31 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op } id = id + "_" + string(uuid.NewUUID()) - return &ResourceLock{ + return &resourceLock{ id: id, record: resourcelock.LeaderElectionRecord{ HolderIdentity: id, - LeaseDurationSeconds: 15, + LeaseDurationSeconds: 1, AcquireTime: metav1.NewTime(time.Now()), - RenewTime: metav1.NewTime(time.Now().Add(15 * time.Second)), + RenewTime: metav1.NewTime(time.Now().Add(1 * time.Second)), LeaderTransitions: 1, }, }, nil } -// ResourceLock implements the ResourceLockInterface. +var _ ControllableResourceLockInterface = &resourceLock{} + +// resourceLock implements the ResourceLockInterface. // By default returns that the current identity holds the lock. -type ResourceLock struct { +type resourceLock struct { id string record resourcelock.LeaderElectionRecord + + blockedLeaderElection atomic.Bool } // Get implements the ResourceLockInterface. -func (f *ResourceLock) Get(ctx context.Context) (*resourcelock.LeaderElectionRecord, []byte, error) { +func (f *resourceLock) Get(ctx context.Context) (*resourcelock.LeaderElectionRecord, []byte, error) { recordBytes, err := json.Marshal(f.record) if err != nil { return nil, nil, err @@ -69,28 +88,49 @@ func (f *ResourceLock) Get(ctx context.Context) (*resourcelock.LeaderElectionRec } // Create implements the ResourceLockInterface. -func (f *ResourceLock) Create(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { +func (f *resourceLock) Create(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { + if f.blockedLeaderElection.Load() { + // If leader election is blocked, we do not allow creating a new record. + return errors.New("leader election is blocked, cannot create new record") + } + f.record = ler return nil } // Update implements the ResourceLockInterface. -func (f *ResourceLock) Update(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { +func (f *resourceLock) Update(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { + if f.blockedLeaderElection.Load() { + // If leader election is blocked, we do not allow updating records + return errors.New("leader election is blocked, cannot update record") + } + f.record = ler + return nil } // RecordEvent implements the ResourceLockInterface. -func (f *ResourceLock) RecordEvent(s string) { +func (f *resourceLock) RecordEvent(s string) { } // Identity implements the ResourceLockInterface. -func (f *ResourceLock) Identity() string { +func (f *resourceLock) Identity() string { return f.id } // Describe implements the ResourceLockInterface. -func (f *ResourceLock) Describe() string { +func (f *resourceLock) Describe() string { return f.id } + +// BlockLeaderElection blocks the leader election process when called. +func (f *resourceLock) BlockLeaderElection() { + f.blockedLeaderElection.Store(true) +} + +// UnblockLeaderElection blocks the leader election process when called. +func (f *resourceLock) UnblockLeaderElection() { + f.blockedLeaderElection.Store(false) +} diff --git a/pkg/leaderelection/leader_election.go b/pkg/leaderelection/leader_election.go index 3dedd462f3..6c013e7992 100644 --- a/pkg/leaderelection/leader_election.go +++ b/pkg/leaderelection/leader_election.go @@ -19,14 +19,15 @@ package leaderelection import ( "errors" "fmt" - "io/ioutil" "os" + "time" "k8s.io/apimachinery/pkg/util/uuid" coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" + "sigs.k8s.io/controller-runtime/pkg/recorder" ) @@ -39,7 +40,7 @@ type Options struct { LeaderElection bool // LeaderElectionResourceLock determines which resource lock to use for leader election, - // defaults to "configmapsleases". + // defaults to "leases". LeaderElectionResourceLock string // LeaderElectionNamespace determines the namespace in which the leader @@ -49,6 +50,16 @@ type Options struct { // LeaderElectionID determines the name of the resource that leader election // will use for holding the leader lock. LeaderElectionID string + + // RenewDeadline is the renew deadline for this leader election client. + // Must be set to ensure the resource lock has an appropriate client timeout. + // Without that, a single slow response from the API server can result + // in losing leadership. + RenewDeadline time.Duration + + // LeaderLabels are an optional set of labels that will be set on the lease object + // when this replica becomes leader + LeaderLabels map[string]string } // NewResourceLock creates a new resource lock for use in a leader election loop. @@ -56,12 +67,12 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op if !options.LeaderElection { return nil, nil } - - // Default resource lock to "configmapsleases". We must keep this default until we are sure all controller-runtime - // users have upgraded from the original default ConfigMap lock to a controller-runtime version that has this new - // default. Many users of controller-runtime skip versions, so we should be extremely conservative here. + // Default resource lock to "leases". The previous default (from v0.7.0 to v0.11.x) was configmapsleases, which was + // used to migrate from configmaps to leases. Since the default was "configmapsleases" for over a year, spanning + // five minor releases, any actively maintained operators are very likely to have a released version that uses + // "configmapsleases". Therefore defaulting to "leases" should be safe. if options.LeaderElectionResourceLock == "" { - options.LeaderElectionResourceLock = resourcelock.ConfigMapsLeasesResourceLock + options.LeaderElectionResourceLock = resourcelock.LeasesResourceLock } // LeaderElectionID must be provided to prevent clashes @@ -85,8 +96,21 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op } id = id + "_" + string(uuid.NewUUID()) + // Construct config for leader election + config = rest.AddUserAgent(config, "leader-election") + + // Timeout set for a client used to contact to Kubernetes should be lower than + // RenewDeadline to keep a single hung request from forcing a leader loss. + // Setting it to max(time.Second, RenewDeadline/2) as a reasonable heuristic. + if options.RenewDeadline != 0 { + timeout := options.RenewDeadline / 2 + if timeout < time.Second { + timeout = time.Second + } + config.Timeout = timeout + } + // Construct clients for leader election - rest.AddUserAgent(config, "leader-election") corev1Client, err := corev1client.NewForConfig(config) if err != nil { return nil, err @@ -97,7 +121,7 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op return nil, err } - return resourcelock.New(options.LeaderElectionResourceLock, + return resourcelock.NewWithLabels(options.LeaderElectionResourceLock, options.LeaderElectionNamespace, options.LeaderElectionID, corev1Client, @@ -105,7 +129,9 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op resourcelock.ResourceLockConfig{ Identity: id, EventRecorder: recorderProvider.GetEventRecorderFor(id), - }) + }, + options.LeaderLabels, + ) } func getInClusterNamespace() (string, error) { @@ -118,7 +144,7 @@ func getInClusterNamespace() (string, error) { } // Load the namespace file and return its content - namespace, err := ioutil.ReadFile(inClusterNamespacePath) + namespace, err := os.ReadFile(inClusterNamespacePath) if err != nil { return "", fmt.Errorf("error reading namespace file: %w", err) } diff --git a/pkg/log/deleg.go b/pkg/log/deleg.go index 9d73947dac..6eb551d3b6 100644 --- a/pkg/log/deleg.go +++ b/pkg/log/deleg.go @@ -25,16 +25,15 @@ import ( // loggerPromise knows how to populate a concrete logr.Logger // with options, given an actual base logger later on down the line. type loggerPromise struct { - logger *DelegatingLogger + logger *delegatingLogSink childPromises []*loggerPromise promisesLock sync.Mutex - name *string - tags []interface{} - level int + name *string + tags []interface{} } -func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromise { +func (p *loggerPromise) WithName(l *delegatingLogSink, name string) *loggerPromise { res := &loggerPromise{ logger: l, name: &name, @@ -48,7 +47,7 @@ func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromis } // WithValues provides a new Logger with the tags appended. -func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *loggerPromise { +func (p *loggerPromise) WithValues(l *delegatingLogSink, tags ...interface{}) *loggerPromise { res := &loggerPromise{ logger: l, tags: tags, @@ -61,61 +60,58 @@ func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *lo return res } -func (p *loggerPromise) V(l *DelegatingLogger, level int) *loggerPromise { - res := &loggerPromise{ - logger: l, - level: level, - promisesLock: sync.Mutex{}, - } - - p.promisesLock.Lock() - defer p.promisesLock.Unlock() - p.childPromises = append(p.childPromises, res) - return res -} - // Fulfill instantiates the Logger with the provided logger. -func (p *loggerPromise) Fulfill(parentLogger logr.Logger) { - logger := logr.WithCallDepth(parentLogger, 1) +func (p *loggerPromise) Fulfill(parentLogSink logr.LogSink) { + sink := parentLogSink if p.name != nil { - logger = logger.WithName(*p.name) + sink = sink.WithName(*p.name) } if p.tags != nil { - logger = logger.WithValues(p.tags...) - } - if p.level != 0 { - logger = logger.V(p.level) + sink = sink.WithValues(p.tags...) } p.logger.lock.Lock() - p.logger.logger = logger + p.logger.logger = sink + if withCallDepth, ok := sink.(logr.CallDepthLogSink); ok { + p.logger.logger = withCallDepth.WithCallDepth(1) + } p.logger.promise = nil p.logger.lock.Unlock() for _, childPromise := range p.childPromises { - childPromise.Fulfill(logger) + childPromise.Fulfill(sink) } } -// DelegatingLogger is a logr.Logger that delegates to another logr.Logger. +// delegatingLogSink is a logsink that delegates to another logr.LogSink. // If the underlying promise is not nil, it registers calls to sub-loggers with // the logging factory to be populated later, and returns a new delegating // logger. It expects to have *some* logr.Logger set at all times (generally // a no-op logger before the promises are fulfilled). -type DelegatingLogger struct { +type delegatingLogSink struct { lock sync.RWMutex - logger logr.Logger + logger logr.LogSink promise *loggerPromise + info logr.RuntimeInfo +} + +// Init implements logr.LogSink. +func (l *delegatingLogSink) Init(info logr.RuntimeInfo) { + eventuallyFulfillRoot() + l.lock.Lock() + defer l.lock.Unlock() + l.info = info } // Enabled tests whether this Logger is enabled. For example, commandline // flags might be used to set the logging verbosity and disable some info // logs. -func (l *DelegatingLogger) Enabled() bool { +func (l *delegatingLogSink) Enabled(level int) bool { + eventuallyFulfillRoot() l.lock.RLock() defer l.lock.RUnlock() - return l.logger.Enabled() + return l.logger.Enabled(level) } // Info logs a non-error message with the given key/value pairs as context. @@ -124,10 +120,11 @@ func (l *DelegatingLogger) Enabled() bool { // the log line. The key/value pairs can then be used to add additional // variable information. The key/value pairs should alternate string // keys and arbitrary values. -func (l *DelegatingLogger) Info(msg string, keysAndValues ...interface{}) { +func (l *delegatingLogSink) Info(level int, msg string, keysAndValues ...interface{}) { + eventuallyFulfillRoot() l.lock.RLock() defer l.lock.RUnlock() - l.logger.Info(msg, keysAndValues...) + l.logger.Info(level, msg, keysAndValues...) } // Error logs an error, with the given message and key/value pairs as context. @@ -138,41 +135,28 @@ func (l *DelegatingLogger) Info(msg string, keysAndValues ...interface{}) { // The msg field should be used to add context to any underlying error, // while the err field should be used to attach the actual error that // triggered this log line, if present. -func (l *DelegatingLogger) Error(err error, msg string, keysAndValues ...interface{}) { +func (l *delegatingLogSink) Error(err error, msg string, keysAndValues ...interface{}) { + eventuallyFulfillRoot() l.lock.RLock() defer l.lock.RUnlock() l.logger.Error(err, msg, keysAndValues...) } -// V returns an Logger value for a specific verbosity level, relative to -// this Logger. In other words, V values are additive. V higher verbosity -// level means a log message is less important. It's illegal to pass a log -// level less than zero. -func (l *DelegatingLogger) V(level int) logr.Logger { - l.lock.RLock() - defer l.lock.RUnlock() - - if l.promise == nil { - return l.logger.V(level) - } - - res := &DelegatingLogger{logger: l.logger} - promise := l.promise.V(res, level) - res.promise = promise - - return res -} - // WithName provides a new Logger with the name appended. -func (l *DelegatingLogger) WithName(name string) logr.Logger { +func (l *delegatingLogSink) WithName(name string) logr.LogSink { + eventuallyFulfillRoot() l.lock.RLock() defer l.lock.RUnlock() if l.promise == nil { - return l.logger.WithName(name) + sink := l.logger.WithName(name) + if withCallDepth, ok := sink.(logr.CallDepthLogSink); ok { + sink = withCallDepth.WithCallDepth(-1) + } + return sink } - res := &DelegatingLogger{logger: l.logger} + res := &delegatingLogSink{logger: l.logger} promise := l.promise.WithName(res, name) res.promise = promise @@ -180,15 +164,20 @@ func (l *DelegatingLogger) WithName(name string) logr.Logger { } // WithValues provides a new Logger with the tags appended. -func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger { +func (l *delegatingLogSink) WithValues(tags ...interface{}) logr.LogSink { + eventuallyFulfillRoot() l.lock.RLock() defer l.lock.RUnlock() if l.promise == nil { - return l.logger.WithValues(tags...) + sink := l.logger.WithValues(tags...) + if withCallDepth, ok := sink.(logr.CallDepthLogSink); ok { + sink = withCallDepth.WithCallDepth(-1) + } + return sink } - res := &DelegatingLogger{logger: l.logger} + res := &delegatingLogSink{logger: l.logger} promise := l.promise.WithValues(res, tags...) res.promise = promise @@ -198,16 +187,19 @@ func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger { // Fulfill switches the logger over to use the actual logger // provided, instead of the temporary initial one, if this method // has not been previously called. -func (l *DelegatingLogger) Fulfill(actual logr.Logger) { +func (l *delegatingLogSink) Fulfill(actual logr.LogSink) { + if actual == nil { + actual = NullLogSink{} + } if l.promise != nil { l.promise.Fulfill(actual) } } -// NewDelegatingLogger constructs a new DelegatingLogger which uses -// the given logger before it's promise is fulfilled. -func NewDelegatingLogger(initial logr.Logger) *DelegatingLogger { - l := &DelegatingLogger{ +// newDelegatingLogSink constructs a new DelegatingLogSink which uses +// the given logger before its promise is fulfilled. +func newDelegatingLogSink(initial logr.LogSink) *delegatingLogSink { + l := &delegatingLogSink{ logger: initial, promise: &loggerPromise{promisesLock: sync.Mutex{}}, } diff --git a/pkg/log/log.go b/pkg/log/log.go index 229ac7ec35..ade21d6fb5 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -17,7 +17,7 @@ limitations under the License. // Package log contains utilities for fetching a new logger // when one is not already available. // -// The Log Handle +// # The Log Handle // // This package contains a root logr.Logger Log. It may be used to // get a handle to whatever the root logging implementation is. By @@ -25,17 +25,21 @@ limitations under the License. // to loggers. When the implementation is set using SetLogger, these // "promises" will be converted over to real loggers. // -// Logr +// # Logr // // All logging in controller-runtime is structured, using a set of interfaces // defined by a package called logr -// (https://godoc.org/github.com/go-logr/logr). The sub-package zap provides +// (https://pkg.go.dev/github.com/go-logr/logr). The sub-package zap provides // helpers for setting up logr backed by Zap (go.uber.org/zap). package log import ( + "bytes" "context" - "sync" + "fmt" + "os" + "runtime/debug" + "sync/atomic" "time" "github.com/go-logr/logr" @@ -43,49 +47,51 @@ import ( // SetLogger sets a concrete logging implementation for all deferred Loggers. func SetLogger(l logr.Logger) { - loggerWasSetLock.Lock() - defer loggerWasSetLock.Unlock() - - loggerWasSet = true - Log.Fulfill(l) + logFullfilled.Store(true) + rootLog.Fulfill(l.GetSink()) } -// It is safe to assume that if this wasn't set within the first 30 seconds of a binaries -// lifetime, it will never get set. The DelegatingLogger causes a high number of memory -// allocations when not given an actual Logger, so we set a NullLogger to avoid that. -// -// We need to keep the DelegatingLogger because we have various inits() that get a logger from -// here. They will always get executed before any code that imports controller-runtime -// has a chance to run and hence to set an actual logger. -func init() { - // Init is blocking, so start a new goroutine - go func() { - time.Sleep(30 * time.Second) - loggerWasSetLock.Lock() - defer loggerWasSetLock.Unlock() - if !loggerWasSet { - Log.Fulfill(NullLogger{}) +func eventuallyFulfillRoot() { + if logFullfilled.Load() { + return + } + if time.Since(rootLogCreated).Seconds() >= 30 { + if logFullfilled.CompareAndSwap(false, true) { + stack := debug.Stack() + stackLines := bytes.Count(stack, []byte{'\n'}) + sep := []byte{'\n', '\t', '>', ' ', ' '} + + fmt.Fprintf(os.Stderr, + "[controller-runtime] log.SetLogger(...) was never called; logs will not be displayed.\nDetected at:%s%s", sep, + // prefix every line, so it's clear this is a stack trace related to the above message + bytes.Replace(stack, []byte{'\n'}, sep, stackLines-1), + ) + SetLogger(logr.New(NullLogSink{})) } - }() + } } var ( - loggerWasSetLock sync.Mutex - loggerWasSet bool + logFullfilled atomic.Bool ) // Log is the base logger used by kubebuilder. It delegates // to another logr.Logger. You *must* call SetLogger to // get any actual logging. If SetLogger is not called within // the first 30 seconds of a binaries lifetime, it will get -// set to a NullLogger. -var Log = NewDelegatingLogger(NullLogger{}) +// set to a NullLogSink. +var ( + rootLog, rootLogCreated = func() (*delegatingLogSink, time.Time) { + return newDelegatingLogSink(NullLogSink{}), time.Now() + }() + Log = logr.New(rootLog) +) // FromContext returns a logger with predefined values from a context.Context. func FromContext(ctx context.Context, keysAndValues ...interface{}) logr.Logger { - var log logr.Logger = Log + log := Log if ctx != nil { - if logger := logr.FromContext(ctx); logger != nil { + if logger, err := logr.FromContext(ctx); err == nil { log = logger } } diff --git a/pkg/log/log_suite_test.go b/pkg/log/log_suite_test.go index bf8e967cb7..f0e349aa86 100644 --- a/pkg/log/log_suite_test.go +++ b/pkg/log/log_suite_test.go @@ -19,13 +19,11 @@ package log import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Log Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Log Suite") } diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go index be3ad87997..404a859e4d 100644 --- a/pkg/log/log_test.go +++ b/pkg/log/log_test.go @@ -17,15 +17,14 @@ limitations under the License. package log import ( - "context" "errors" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -var _ logr.Logger = &DelegatingLogger{} +var _ logr.LogSink = &delegatingLogSink{} // logInfo is the information for a particular fakeLogger message. type logInfo struct { @@ -49,7 +48,10 @@ type fakeLogger struct { root *fakeLoggerRoot } -func (f *fakeLogger) WithName(name string) logr.Logger { +func (f *fakeLogger) Init(info logr.RuntimeInfo) { +} + +func (f *fakeLogger) WithName(name string) logr.LogSink { names := append([]string(nil), f.name...) names = append(names, name) return &fakeLogger{ @@ -59,7 +61,7 @@ func (f *fakeLogger) WithName(name string) logr.Logger { } } -func (f *fakeLogger) WithValues(vals ...interface{}) logr.Logger { +func (f *fakeLogger) WithValues(vals ...interface{}) logr.LogSink { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) return &fakeLogger{ @@ -80,7 +82,7 @@ func (f *fakeLogger) Error(err error, msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Info(msg string, vals ...interface{}) { +func (f *fakeLogger) Info(level int, msg string, vals ...interface{}) { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) f.root.messages = append(f.root.messages, logInfo{ @@ -90,8 +92,7 @@ func (f *fakeLogger) Info(msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Enabled() bool { return true } -func (f *fakeLogger) V(lvl int) logr.Logger { return f } +func (f *fakeLogger) Enabled(level int) bool { return true } var _ = Describe("logging", func() { @@ -103,7 +104,7 @@ var _ = Describe("logging", func() { By("actually setting the logger") logger := &fakeLogger{root: &fakeLoggerRoot{}} - SetLogger(logger) + SetLogger(logr.New(logger)) By("grabbing another sub-logger and logging to both loggers") l2 := Log.WithName("runtimeLog").WithValues("newtag", "newvalue2") @@ -121,24 +122,24 @@ var _ = Describe("logging", func() { Describe("lazy logger initialization", func() { var ( root *fakeLoggerRoot - baseLog logr.Logger - delegLog *DelegatingLogger + baseLog logr.LogSink + delegLog *delegatingLogSink ) BeforeEach(func() { root = &fakeLoggerRoot{} baseLog = &fakeLogger{root: root} - delegLog = NewDelegatingLogger(NullLogger{}) + delegLog = newDelegatingLogSink(NullLogSink{}) }) It("should delegate with name", func() { By("asking for a logger with a name before fulfill, and logging") - befFulfill1 := delegLog.WithName("before-fulfill") + befFulfill1 := logr.New(delegLog).WithName("before-fulfill") befFulfill2 := befFulfill1.WithName("two") befFulfill1.Info("before fulfill") By("logging on the base logger before fulfill") - delegLog.Info("before fulfill base") + logr.New(delegLog).Info("before fulfill base") By("ensuring that no messages were actually recorded") Expect(root.messages).To(BeEmpty()) @@ -154,7 +155,7 @@ var _ = Describe("logging", func() { befFulfill1.WithName("after-from-before").Info("after 3") By("logging with new loggers") - delegLog.WithName("after-fulfill").Info("after 4") + logr.New(delegLog).WithName("after-fulfill").Info("after 4") By("ensuring that the messages are appropriately named") Expect(root.messages).To(ConsistOf( @@ -179,10 +180,10 @@ var _ = Describe("logging", func() { // Constructing the child in the goroutine does not reliably // trigger the race detector - child := delegLog.WithName("child") + child := logr.New(delegLog).WithName("child") go func() { defer GinkgoRecover() - delegLog.Fulfill(NullLogger{}) + delegLog.Fulfill(NullLogSink{}) close(fulfillDone) }() go func() { @@ -192,22 +193,22 @@ var _ = Describe("logging", func() { }() go func() { defer GinkgoRecover() - delegLog.WithValues("with-value") + delegLog.WithValues("key", "with-value") close(withValuesDone) }() go func() { defer GinkgoRecover() - child.WithValues("grandchild") + child.WithValues("key", "grandchild") close(grandChildDone) }() go func() { defer GinkgoRecover() - delegLog.Enabled() + logr.New(delegLog).Enabled() close(logEnabledDone) }() go func() { defer GinkgoRecover() - delegLog.Info("hello world") + logr.New(delegLog).Info("hello world") close(logInfoDone) }() go func() { @@ -217,7 +218,7 @@ var _ = Describe("logging", func() { }() go func() { defer GinkgoRecover() - delegLog.V(1) + logr.New(delegLog).V(1) close(logVDone) }() @@ -233,12 +234,12 @@ var _ = Describe("logging", func() { It("should delegate with tags", func() { By("asking for a logger with a name before fulfill, and logging") - befFulfill1 := delegLog.WithValues("tag1", "val1") + befFulfill1 := logr.New(delegLog).WithValues("tag1", "val1") befFulfill2 := befFulfill1.WithValues("tag2", "val2") befFulfill1.Info("before fulfill") By("logging on the base logger before fulfill") - delegLog.Info("before fulfill base") + logr.New(delegLog).Info("before fulfill base") By("ensuring that no messages were actually recorded") Expect(root.messages).To(BeEmpty()) @@ -254,7 +255,7 @@ var _ = Describe("logging", func() { befFulfill1.WithValues("tag3", "val3").Info("after 3") By("logging with new loggers") - delegLog.WithValues("tag3", "val3").Info("after 4") + logr.New(delegLog).WithValues("tag3", "val3").Info("after 4") By("ensuring that the messages are appropriately named") Expect(root.messages).To(ConsistOf( @@ -270,13 +271,13 @@ var _ = Describe("logging", func() { delegLog.Fulfill(baseLog) By("logging a bit") - delegLog.Info("msg 1") + logr.New(delegLog).Info("msg 1") By("fulfilling with a new logger") delegLog.Fulfill(&fakeLogger{}) By("logging some more") - delegLog.Info("msg 2") + logr.New(delegLog).Info("msg 2") By("checking that all log messages are present") Expect(root.messages).To(ConsistOf( @@ -284,20 +285,28 @@ var _ = Describe("logging", func() { logInfo{msg: "msg 2"}, )) }) + + It("should handle nil sinks", func() { + By("fulfilling once") + delegLog.Fulfill(logr.Discard().GetSink()) + By("grabbing a sub-logger and logging") + l1 := logr.New(delegLog).WithName("nilsink").WithValues("newtag", "newvalue2") + l1.Info("test") + }) }) Describe("logger from context", func() { - It("should return default logger when context is empty", func() { - gotLog := FromContext(context.Background()) + It("should return default logger when context is empty", func(ctx SpecContext) { + gotLog := FromContext(ctx) Expect(gotLog).To(Not(BeNil())) }) - It("should return existing logger", func() { + It("should return existing logger", func(specCtx SpecContext) { root := &fakeLoggerRoot{} baseLog := &fakeLogger{root: root} - wantLog := baseLog.WithName("my-logger") - ctx := IntoContext(context.Background(), wantLog) + wantLog := logr.New(baseLog).WithName("my-logger") + ctx := IntoContext(specCtx, wantLog) gotLog := FromContext(ctx) Expect(gotLog).To(Not(BeNil())) @@ -308,12 +317,12 @@ var _ = Describe("logging", func() { )) }) - It("should have added key-values", func() { + It("should have added key-values", func(specCtx SpecContext) { root := &fakeLoggerRoot{} baseLog := &fakeLogger{root: root} - wantLog := baseLog.WithName("my-logger") - ctx := IntoContext(context.Background(), wantLog) + wantLog := logr.New(baseLog).WithName("my-logger") + ctx := IntoContext(specCtx, wantLog) gotLog := FromContext(ctx, "tag1", "value1") Expect(gotLog).To(Not(BeNil())) diff --git a/pkg/log/null.go b/pkg/log/null.go index 09a5a02eb6..f3e81074fe 100644 --- a/pkg/log/null.go +++ b/pkg/log/null.go @@ -24,37 +24,36 @@ import ( // but avoids accidentally adding the testing flags to // all binaries. -// NullLogger is a logr.Logger that does nothing. -type NullLogger struct{} +// NullLogSink is a logr.Logger that does nothing. +type NullLogSink struct{} -var _ logr.Logger = NullLogger{} +var _ logr.LogSink = NullLogSink{} + +// Init implements logr.LogSink. +func (log NullLogSink) Init(logr.RuntimeInfo) { +} // Info implements logr.InfoLogger. -func (NullLogger) Info(_ string, _ ...interface{}) { +func (NullLogSink) Info(_ int, _ string, _ ...interface{}) { // Do nothing. } // Enabled implements logr.InfoLogger. -func (NullLogger) Enabled() bool { +func (NullLogSink) Enabled(level int) bool { return false } // Error implements logr.Logger. -func (NullLogger) Error(_ error, _ string, _ ...interface{}) { +func (NullLogSink) Error(_ error, _ string, _ ...interface{}) { // Do nothing. } -// V implements logr.Logger. -func (log NullLogger) V(_ int) logr.Logger { - return log -} - // WithName implements logr.Logger. -func (log NullLogger) WithName(_ string) logr.Logger { +func (log NullLogSink) WithName(_ string) logr.LogSink { return log } // WithValues implements logr.Logger. -func (log NullLogger) WithValues(_ ...interface{}) logr.Logger { +func (log NullLogSink) WithValues(_ ...interface{}) logr.LogSink { return log } diff --git a/pkg/log/warning_handler.go b/pkg/log/warning_handler.go index 3012fdd411..413b56d2e4 100644 --- a/pkg/log/warning_handler.go +++ b/pkg/log/warning_handler.go @@ -17,13 +17,12 @@ limitations under the License. package log import ( + "context" "sync" - - "github.com/go-logr/logr" ) // KubeAPIWarningLoggerOptions controls the behavior -// of a rest.WarningHandler constructed using NewKubeAPIWarningLogger(). +// of a rest.WarningHandlerWithContext constructed using NewKubeAPIWarningLogger(). type KubeAPIWarningLoggerOptions struct { // Deduplicate indicates a given warning message should only be written once. // Setting this to true in a long-running process handling many warnings can @@ -33,10 +32,8 @@ type KubeAPIWarningLoggerOptions struct { // KubeAPIWarningLogger is a wrapper around // a provided logr.Logger that implements the -// rest.WarningHandler interface. +// rest.WarningHandlerWithContext interface. type KubeAPIWarningLogger struct { - // logger is used to log responses with the warning header - logger logr.Logger // opts contain options controlling warning output opts KubeAPIWarningLoggerOptions // writtenLock gurads written @@ -46,9 +43,11 @@ type KubeAPIWarningLogger struct { written map[string]struct{} } -// HandleWarningHeader handles logging for responses from API server that are -// warnings with code being 299 and uses a logr.Logger for it's logging purposes. -func (l *KubeAPIWarningLogger) HandleWarningHeader(code int, agent string, message string) { +// HandleWarningHeaderWithContext handles logging for responses from API server that are +// warnings with code being 299 and uses a logr.Logger from context for its logging purposes. +func (l *KubeAPIWarningLogger) HandleWarningHeaderWithContext(ctx context.Context, code int, _ string, message string) { + log := FromContext(ctx) + if code != 299 || len(message) == 0 { return } @@ -62,13 +61,13 @@ func (l *KubeAPIWarningLogger) HandleWarningHeader(code int, agent string, messa } l.written[message] = struct{}{} } - l.logger.Info(message) + log.Info(message) } -// NewKubeAPIWarningLogger returns an implementation of rest.WarningHandler that logs warnings -// with code = 299 to the provided logr.Logger. -func NewKubeAPIWarningLogger(l logr.Logger, opts KubeAPIWarningLoggerOptions) *KubeAPIWarningLogger { - h := &KubeAPIWarningLogger{logger: l, opts: opts} +// NewKubeAPIWarningLogger returns an implementation of rest.WarningHandlerWithContext that logs warnings +// with code = 299 to the logger passed into HandleWarningHeaderWithContext via the context. +func NewKubeAPIWarningLogger(opts KubeAPIWarningLoggerOptions) *KubeAPIWarningLogger { + h := &KubeAPIWarningLogger{opts: opts} if opts.Deduplicate { h.written = map[string]struct{}{} } diff --git a/pkg/log/zap/flags.go b/pkg/log/zap/flags.go index 3339655075..4ebac57dcb 100644 --- a/pkg/log/zap/flags.go +++ b/pkg/log/zap/flags.go @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package zap contains helpers for setting up a new logr.Logger instance -// using the Zap logging framework. package zap import ( @@ -32,6 +30,7 @@ var levelStrings = map[string]zapcore.Level{ "debug": zap.DebugLevel, "info": zap.InfoLevel, "error": zap.ErrorLevel, + "panic": zap.PanicLevel, } var stackLevelStrings = map[string]zapcore.Level{ @@ -128,3 +127,41 @@ func (ev *stackTraceFlag) String() string { func (ev *stackTraceFlag) Type() string { return "level" } + +type timeEncodingFlag struct { + setFunc func(zapcore.TimeEncoder) + value string +} + +var _ flag.Value = &timeEncodingFlag{} + +func (ev *timeEncodingFlag) String() string { + return ev.value +} + +func (ev *timeEncodingFlag) Type() string { + return "time-encoding" +} + +func (ev *timeEncodingFlag) Set(flagValue string) error { + val := strings.ToLower(flagValue) + switch val { + case "rfc3339nano": + ev.setFunc(zapcore.RFC3339NanoTimeEncoder) + case "rfc3339": + ev.setFunc(zapcore.RFC3339TimeEncoder) + case "iso8601": + ev.setFunc(zapcore.ISO8601TimeEncoder) + case "millis": + ev.setFunc(zapcore.EpochMillisTimeEncoder) + case "nanos": + ev.setFunc(zapcore.EpochNanosTimeEncoder) + case "epoch": + ev.setFunc(zapcore.EpochTimeEncoder) + default: + return fmt.Errorf("invalid time-encoding value \"%s\"", flagValue) + } + + ev.value = flagValue + return nil +} diff --git a/pkg/log/zap/kube_helpers.go b/pkg/log/zap/kube_helpers.go index 765327d623..c47fe6646f 100644 --- a/pkg/log/zap/kube_helpers.go +++ b/pkg/log/zap/kube_helpers.go @@ -18,12 +18,12 @@ package zap import ( "fmt" + "reflect" "go.uber.org/zap/buffer" "go.uber.org/zap/zapcore" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ) // KubeAwareEncoder is a Kubernetes-aware Zap Encoder. @@ -40,21 +40,6 @@ type KubeAwareEncoder struct { Verbose bool } -// namespacedNameWrapper is a zapcore.ObjectMarshaler for Kubernetes NamespacedName. -type namespacedNameWrapper struct { - types.NamespacedName -} - -func (w namespacedNameWrapper) MarshalLogObject(enc zapcore.ObjectEncoder) error { - if w.Namespace != "" { - enc.AddString("namespace", w.Namespace) - } - - enc.AddString("name", w.Name) - - return nil -} - // kubeObjectWrapper is a zapcore.ObjectMarshaler for Kubernetes objects. type kubeObjectWrapper struct { obj runtime.Object @@ -64,6 +49,11 @@ type kubeObjectWrapper struct { func (w kubeObjectWrapper) MarshalLogObject(enc zapcore.ObjectEncoder) error { // TODO(directxman12): log kind and apiversion if not set explicitly (common case) // -- needs an a scheme to convert to the GVK. + + if reflect.ValueOf(w.obj).IsNil() { + return fmt.Errorf("got nil for runtime.Object") + } + if gvk := w.obj.GetObjectKind().GroupVersionKind(); gvk.Version != "" { enc.AddString("apiVersion", gvk.GroupVersion().String()) enc.AddString("kind", gvk.Kind) @@ -88,6 +78,7 @@ func (w kubeObjectWrapper) MarshalLogObject(enc zapcore.ObjectEncoder) error { func (k *KubeAwareEncoder) Clone() zapcore.Encoder { return &KubeAwareEncoder{ Encoder: k.Encoder.Clone(), + Verbose: k.Verbose, } } @@ -113,12 +104,6 @@ func (k *KubeAwareEncoder) EncodeEntry(entry zapcore.Entry, fields []zapcore.Fie Key: field.Key, Interface: kubeObjectWrapper{obj: val}, } - case types.NamespacedName: - fields[i] = zapcore.Field{ - Type: zapcore.ObjectMarshalerType, - Key: field.Key, - Interface: namespacedNameWrapper{NamespacedName: val}, - } } } } diff --git a/pkg/log/zap/zap.go b/pkg/log/zap/zap.go index 22eb5d771a..607b6680d5 100644 --- a/pkg/log/zap/zap.go +++ b/pkg/log/zap/zap.go @@ -101,7 +101,7 @@ func newConsoleEncoder(opts ...EncoderConfigOption) zapcore.Encoder { return zapcore.NewConsoleEncoder(encoderConfig) } -// Level sets Options.Level, which configures the the minimum enabled logging level e.g Debug, Info. +// Level sets Options.Level, which configures the minimum enabled logging level e.g Debug, Info. // A zap log level should be multiplied by -1 to get the logr verbosity. // For example, to get logr verbosity of 3, pass zapcore.Level(-3) to this Opts. // See https://pkg.go.dev/github.com/go-logr/zapr for how zap level relates to logr verbosity. @@ -138,7 +138,7 @@ type Options struct { // console when Development is true and JSON otherwise Encoder zapcore.Encoder // EncoderConfigOptions can modify the EncoderConfig needed to initialize an Encoder. - // See https://godoc.org/go.uber.org/zap/zapcore#EncoderConfig for the list of options + // See https://pkg.go.dev/go.uber.org/zap/zapcore#EncoderConfig for the list of options // that can be configured. // Note that the EncoderConfigOptions are not applied when the Encoder option is already set. EncoderConfigOptions []EncoderConfigOption @@ -148,11 +148,6 @@ type Options struct { // DestWriter controls the destination of the log output. Defaults to // os.Stderr. DestWriter io.Writer - // DestWritter controls the destination of the log output. Defaults to - // os.Stderr. - // - // Deprecated: Use DestWriter instead - DestWritter io.Writer // Level configures the verbosity of the logging. // Defaults to Debug when Development is true and Info otherwise. // A zap log level should be multiplied by -1 to get the logr verbosity. @@ -167,15 +162,15 @@ type Options struct { // ZapOpts allows passing arbitrary zap.Options to configure on the // underlying Zap logger. ZapOpts []zap.Option + // TimeEncoder specifies the encoder for the timestamps in log messages. + // Defaults to RFC3339TimeEncoder. + TimeEncoder zapcore.TimeEncoder } // addDefaults adds defaults to the Options. func (o *Options) addDefaults() { - if o.DestWriter == nil && o.DestWritter == nil { + if o.DestWriter == nil { o.DestWriter = os.Stderr - } else if o.DestWriter == nil && o.DestWritter != nil { - // while misspelled DestWritter is deprecated but still not removed - o.DestWriter = o.DestWritter } if o.Development { @@ -212,6 +207,16 @@ func (o *Options) addDefaults() { })) } } + + if o.TimeEncoder == nil { + o.TimeEncoder = zapcore.RFC3339TimeEncoder + } + f := func(ecfg *zapcore.EncoderConfig) { + ecfg.EncodeTime = o.TimeEncoder + } + // prepend instead of append it in case someone adds a time encoder option in it + o.EncoderConfigOptions = append([]EncoderConfigOption{f}, o.EncoderConfigOptions...) + if o.Encoder == nil { o.Encoder = o.NewEncoder(o.EncoderConfigOptions...) } @@ -231,19 +236,22 @@ func NewRaw(opts ...Opts) *zap.Logger { // this basically mimics NewConfig, but with a custom sink sink := zapcore.AddSync(o.DestWriter) - o.ZapOpts = append(o.ZapOpts, zap.AddCallerSkip(1), zap.ErrorOutput(sink)) + o.ZapOpts = append(o.ZapOpts, zap.ErrorOutput(sink)) log := zap.New(zapcore.NewCore(&KubeAwareEncoder{Encoder: o.Encoder, Verbose: o.Development}, sink, o.Level)) log = log.WithOptions(o.ZapOpts...) return log } -// BindFlags will parse the given flagset for zap option flags and set the log options accordingly -// zap-devel: Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn) -// Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) -// zap-encoder: Zap log encoding (one of 'json' or 'console') -// zap-log-level: Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', -// or any integer value > 0 which corresponds to custom debug levels of increasing verbosity") -// zap-stacktrace-level: Zap Level at and above which stacktraces are captured (one of 'info', 'error' or 'panic') +// BindFlags will parse the given flagset for zap option flags and set the log options accordingly: +// - zap-devel: +// Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn) +// Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) +// - zap-encoder: Zap log encoding (one of 'json' or 'console') +// - zap-log-level: Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic' +// or any integer value > 0 which corresponds to custom debug levels of increasing verbosity"). +// - zap-stacktrace-level: Zap Level at and above which stacktraces are captured (one of 'info', 'error' or 'panic') +// - zap-time-encoding: Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'), +// Defaults to 'epoch'. func (o *Options) BindFlags(fs *flag.FlagSet) { // Set Development mode value fs.BoolVar(&o.Development, "zap-devel", o.Development, @@ -263,7 +271,7 @@ func (o *Options) BindFlags(fs *flag.FlagSet) { o.Level = fromFlag } fs.Var(&levelVal, "zap-log-level", - "Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', "+ + "Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic'"+ "or any integer value > 0 which corresponds to custom debug levels of increasing verbosity") // Set the StrackTrace Level @@ -273,13 +281,21 @@ func (o *Options) BindFlags(fs *flag.FlagSet) { } fs.Var(&stackVal, "zap-stacktrace-level", "Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic').") + + // Set the time encoding + var timeEncoderVal timeEncodingFlag + timeEncoderVal.setFunc = func(fromFlag zapcore.TimeEncoder) { + o.TimeEncoder = fromFlag + } + fs.Var(&timeEncoderVal, "zap-time-encoding", "Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'.") } // UseFlagOptions configures the logger to use the Options set by parsing zap option flags from the CLI. -// opts := zap.Options{} -// opts.BindFlags(flag.CommandLine) -// flag.Parse() -// log := zap.New(zap.UseFlagOptions(&opts)) +// +// opts := zap.Options{} +// opts.BindFlags(flag.CommandLine) +// flag.Parse() +// log := zap.New(zap.UseFlagOptions(&opts)) func UseFlagOptions(in *Options) Opts { return func(o *Options) { *o = *in diff --git a/pkg/log/zap/zap_suite_test.go b/pkg/log/zap/zap_suite_test.go index 43044d8066..d7a7f22866 100644 --- a/pkg/log/zap/zap_suite_test.go +++ b/pkg/log/zap/zap_suite_test.go @@ -19,13 +19,11 @@ package zap import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Zap Log Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Zap Log Suite") } diff --git a/pkg/log/zap/zap_test.go b/pkg/log/zap/zap_test.go index 095fc54598..3e80113a65 100644 --- a/pkg/log/zap/zap_test.go +++ b/pkg/log/zap/zap_test.go @@ -21,9 +21,10 @@ import ( "encoding/json" "flag" "os" + "reflect" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/zap/zapcore" corev1 "k8s.io/api/core/v1" @@ -44,6 +45,7 @@ type fakeSyncWriter bool func (w *fakeSyncWriter) Write(p []byte) (int, error) { return len(p), nil } + func (w *fakeSyncWriter) Sync() error { *w = true return nil @@ -61,7 +63,7 @@ type fakeLoggerRoot struct { messages []logInfo } -var _ logr.Logger = &fakeLogger{} +var _ logr.LogSink = &fakeLogger{} // fakeLogger is a fake implementation of logr.Logger that records // messages, tags, and names, @@ -73,7 +75,10 @@ type fakeLogger struct { root *fakeLoggerRoot } -func (f *fakeLogger) WithName(name string) logr.Logger { +func (f *fakeLogger) Init(info logr.RuntimeInfo) { +} + +func (f *fakeLogger) WithName(name string) logr.LogSink { names := append([]string(nil), f.name...) names = append(names, name) return &fakeLogger{ @@ -83,7 +88,7 @@ func (f *fakeLogger) WithName(name string) logr.Logger { } } -func (f *fakeLogger) WithValues(vals ...interface{}) logr.Logger { +func (f *fakeLogger) WithValues(vals ...interface{}) logr.LogSink { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) return &fakeLogger{ @@ -104,7 +109,7 @@ func (f *fakeLogger) Error(err error, msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Info(msg string, vals ...interface{}) { +func (f *fakeLogger) Info(level int, msg string, vals ...interface{}) { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) f.root.messages = append(f.root.messages, logInfo{ @@ -114,8 +119,8 @@ func (f *fakeLogger) Info(msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Enabled() bool { return true } -func (f *fakeLogger) V(lvl int) logr.Logger { return f } +func (f *fakeLogger) Enabled(level int) bool { return true } +func (f *fakeLogger) V(lvl int) logr.LogSink { return f } var _ = Describe("Zap options setup", func() { var opts *Options @@ -251,6 +256,14 @@ var _ = Describe("Zap logger setup", func() { "namespace": name.Namespace, })) }) + + It("should not panic with nil obj", func() { + var pod *corev1.Pod + logger.Info("here's a kubernetes object", "thing", pod) + + outRaw := logOut.Bytes() + Expect(string(outRaw)).Should(ContainSubstring("got nil for runtime.Object")) + }) } Context("with logger created using New", func() { @@ -261,7 +274,6 @@ var _ = Describe("Zap logger setup", func() { logger = New(WriteTo(logOut), UseDevMode(false)) }) defineTests() - }) }) }) @@ -297,7 +309,6 @@ var _ = Describe("Zap log level flag options setup", func() { Expect(string(outRaw)).Should(ContainSubstring(logInfoLevel0)) Expect(string(outRaw)).Should(ContainSubstring(logDebugLevel1)) - }) It("Should output only error logs, otherwise empty logs", func() { @@ -317,6 +328,23 @@ var _ = Describe("Zap log level flag options setup", func() { Expect(outRaw).To(BeEmpty()) }) + It("Should output only panic logs, otherwise empty logs", func() { + args := []string{"--zap-log-level=panic"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) + + logOut := new(bytes.Buffer) + + logger := New(UseFlagOptions(&fromFlags), WriteTo(logOut)) + logger.V(0).Info(logInfoLevel0) + logger.V(1).Info(logDebugLevel1) + logger.V(2).Info(logDebugLevel2) + + outRaw := logOut.Bytes() + + Expect(outRaw).To(BeEmpty()) + }) }) Context("with zap-log-level with increased verbosity.", func() { @@ -373,7 +401,6 @@ var _ = Describe("Zap log level flag options setup", func() { Expect(string(outRaw)).Should(ContainSubstring(logDebugLevel1)) Expect(string(outRaw)).Should(ContainSubstring(logDebugLevel2)) Expect(string(outRaw)).Should(ContainSubstring(logDebugLevel3)) - }) It("Should output info, and debug logs with increased verbosity, and with production mode set to true.", func() { args := []string{"--zap-log-level=3", "--zap-devel=true"} @@ -394,13 +421,10 @@ var _ = Describe("Zap log level flag options setup", func() { Expect(string(outRaw)).Should(ContainSubstring(logDebugLevel1)) Expect(string(outRaw)).Should(ContainSubstring(logDebugLevel2)) Expect(string(outRaw)).Should(ContainSubstring(logDebugLevel3)) - }) - }) Context("with zap-stacktrace-level options provided", func() { - It("Should output stacktrace at info level, with development mode set to true.", func() { args := []string{"--zap-stacktrace-level=info", "--zap-devel=true"} fromFlags.BindFlags(&fs) @@ -435,7 +459,6 @@ var _ = Describe("Zap log level flag options setup", func() { Expect(out.StacktraceLevel.Enabled(zapcore.ErrorLevel)).To(BeFalse()) Expect(out.StacktraceLevel.Enabled(zapcore.InfoLevel)).To(BeFalse()) }) - }) Context("with only -zap-devel flag provided", func() { @@ -468,39 +491,76 @@ var _ = Describe("Zap log level flag options setup", func() { Expect(out.Level).To(BeNil()) Expect(out.StacktraceLevel).To(BeNil()) Expect(out.EncoderConfigOptions).To(BeNil()) - }) }) - Context("with encoder options provided programmatically", func() { + Context("with zap-time-encoding flag provided", func() { + It("Should set time encoder in options", func() { + args := []string{"--zap-time-encoding=rfc3339"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) + + opt := Options{} + UseFlagOptions(&fromFlags)(&opt) + opt.addDefaults() - It("Should set Console Encoder, with given Nanos TimeEncoder option.", func() { + optVal := reflect.ValueOf(opt.TimeEncoder) + expVal := reflect.ValueOf(zapcore.RFC3339TimeEncoder) + + Expect(optVal.Pointer()).To(Equal(expVal.Pointer())) + }) + + It("Should default to 'rfc3339' time encoding", func() { + args := []string{""} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) + + opt := Options{} + UseFlagOptions(&fromFlags)(&opt) + opt.addDefaults() + + optVal := reflect.ValueOf(opt.TimeEncoder) + expVal := reflect.ValueOf(zapcore.RFC3339TimeEncoder) + + Expect(optVal.Pointer()).To(Equal(expVal.Pointer())) + }) + + It("Should return an error message, with unknown time-encoding", func() { + fs = *flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + args := []string{"--zap-time-encoding=foobar"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).To(HaveOccurred()) + }) + + It("Should propagate time encoder to logger", func() { + // zaps ISO8601TimeEncoder uses 2006-01-02T15:04:05.000Z0700 as pattern for iso8601 encoding + iso8601Pattern := `^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}([-+][0-9]{4}|Z)` + + args := []string{"--zap-time-encoding=iso8601"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) logOut := new(bytes.Buffer) - f := func(ec *zapcore.EncoderConfig) { - if err := ec.EncodeTime.UnmarshalText([]byte("nanos")); err != nil { - Expect(err).ToNot(HaveOccurred()) - } - } - opts := func(o *Options) { - o.EncoderConfigOptions = append(o.EncoderConfigOptions, f) - } - log := New(UseDevMode(true), WriteTo(logOut), opts) - log.Info("This is a test message") + + logger := New(UseFlagOptions(&fromFlags), WriteTo(logOut)) + logger.Info("This is a test message") + outRaw := logOut.Bytes() - // Assert for Console Encoder - res := map[string]interface{}{} - Expect(json.Unmarshal(outRaw, &res)).ToNot(Succeed()) - // Assert for Epoch Nanos TimeEncoder - Expect(string(outRaw)).ShouldNot(ContainSubstring(".")) + res := map[string]interface{}{} + Expect(json.Unmarshal(outRaw, &res)).To(Succeed()) + Expect(res["ts"]).Should(MatchRegexp(iso8601Pattern)) }) + }) + + Context("with encoder options provided programmatically", func() { It("Should set JSON Encoder, with given Millis TimeEncoder option, and MessageKey", func() { logOut := new(bytes.Buffer) f := func(ec *zapcore.EncoderConfig) { ec.MessageKey = "MillisTimeFormat" - if err := ec.EncodeTime.UnmarshalText([]byte("millis")); err != nil { - Expect(err).ToNot(HaveOccurred()) - } } opts := func(o *Options) { o.EncoderConfigOptions = append(o.EncoderConfigOptions, f) @@ -511,8 +571,6 @@ var _ = Describe("Zap log level flag options setup", func() { // Assert for JSON Encoder res := map[string]interface{}{} Expect(json.Unmarshal(outRaw, &res)).To(Succeed()) - // Assert for Epoch Nanos TimeEncoder - Expect(string(outRaw)).Should(ContainSubstring(".")) // Assert for MessageKey Expect(string(outRaw)).Should(ContainSubstring("MillisTimeFormat")) }) @@ -535,8 +593,8 @@ var _ = Describe("Zap log level flag options setup", func() { logOut.Truncate(0) logger.V(4).Info("test 4") // Should not be logged Expect(logOut.String()).To(BeEmpty()) - logger.V(-3).Info("test -3") // Log a panic, since V(-1*N) for all N > 0 is not permitted. - Expect(logOut.String()).To(ContainSubstring(`"level":"dpanic"`)) + logger.V(-3).Info("test -3") + Expect(logOut.String()).To(ContainSubstring("test -3")) }) It("does not log with positive logr level", func() { By("setting up the logger") diff --git a/pkg/manager/example_test.go b/pkg/manager/example_test.go index 17557d1817..02cfa11946 100644 --- a/pkg/manager/example_test.go +++ b/pkg/manager/example_test.go @@ -20,9 +20,9 @@ import ( "context" "os" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client/config" - conf "sigs.k8s.io/controller-runtime/pkg/config" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -51,7 +51,7 @@ func ExampleNew() { } // This example creates a new Manager that has a cache scoped to a list of namespaces. -func ExampleNew_multinamespaceCache() { +func ExampleNew_limitToNamespaces() { cfg, err := config.GetConfig() if err != nil { log.Error(err, "unable to get kubeconfig") @@ -59,8 +59,14 @@ func ExampleNew_multinamespaceCache() { } mgr, err := manager.New(cfg, manager.Options{ - NewCache: cache.MultiNamespacedCacheBuilder([]string{"namespace1", "namespace2"}), - }) + NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) { + opts.DefaultNamespaces = map[string]cache.Config{ + "namespace1": {}, + "namespace2": {}, + } + return cache.New(config, opts) + }}, + ) if err != nil { log.Error(err, "unable to set up manager") os.Exit(1) @@ -87,43 +93,3 @@ func ExampleManager_start() { os.Exit(1) } } - -// This example will populate Options from a custom config file -// using defaults. -func ExampleOptions_andFrom() { - opts := manager.Options{} - if _, err := opts.AndFrom(conf.File()); err != nil { - log.Error(err, "unable to load config") - os.Exit(1) - } - - cfg, err := config.GetConfig() - if err != nil { - log.Error(err, "unable to get kubeconfig") - os.Exit(1) - } - - mgr, err := manager.New(cfg, opts) - if err != nil { - log.Error(err, "unable to set up manager") - os.Exit(1) - } - log.Info("created manager", "manager", mgr) -} - -// This example will populate Options from a custom config file -// using defaults and will panic if there are errors. -func ExampleOptions_andFromOrDie() { - cfg, err := config.GetConfig() - if err != nil { - log.Error(err, "unable to get kubeconfig") - os.Exit(1) - } - - mgr, err := manager.New(cfg, manager.Options{}.AndFromOrDie(conf.File())) - if err != nil { - log.Error(err, "unable to set up manager") - os.Exit(1) - } - log.Info("created manager", "manager", mgr) -} diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index 7c25bd3c60..a2c3e5324d 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -22,11 +22,12 @@ import ( "fmt" "net" "net/http" + "net/http/pprof" "sync" + "sync/atomic" "time" "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" @@ -38,11 +39,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" - "sigs.k8s.io/controller-runtime/pkg/metrics" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -55,22 +56,20 @@ const ( defaultReadinessEndpoint = "/readyz" defaultLivenessEndpoint = "/healthz" - defaultMetricsEndpoint = "/metrics" ) var _ Runnable = &controllerManager{} type controllerManager struct { - // cluster holds a variety of methods to interact with a cluster. Required. - cluster cluster.Cluster + sync.Mutex + started bool - // leaderElectionRunnables is the set of Controllers that the controllerManager injects deps into and Starts. - // These Runnables are managed by lead election. - leaderElectionRunnables []Runnable + stopProcedureEngaged *int64 + errChan chan error + runnables *runnables - // nonLeaderElectionRunnables is the set of webhook servers that the controllerManager injects deps into and Starts. - // These Runnables will not be blocked by lead election. - nonLeaderElectionRunnables []Runnable + // cluster holds a variety of methods to interact with a cluster. Required. + cluster cluster.Cluster // recorderProvider is used to generate event recorders that will be injected into Controllers // (and EventHandlers, Sources and Predicates). @@ -83,11 +82,8 @@ type controllerManager struct { // on shutdown leaderElectionReleaseOnCancel bool - // metricsListener is used to serve prometheus metrics - metricsListener net.Listener - - // metricsExtraHandlers contains extra handlers to register on http server that serves metrics. - metricsExtraHandlers map[string]http.Handler + // metricsServer is used to serve prometheus metrics + metricsServer metricsserver.Server // healthProbeListener is used to serve liveness probe healthProbeListener net.Listener @@ -104,52 +100,38 @@ type controllerManager struct { // Healthz probe handler healthzHandler *healthz.Handler - mu sync.Mutex - started bool - startedLeader bool - healthzStarted bool - errChan chan error + // pprofListener is used to serve pprof + pprofListener net.Listener - // controllerOptions are the global controller options. - controllerOptions v1alpha1.ControllerConfigurationSpec + // controllerConfig are the global controller options. + controllerConfig config.Controller // Logger is the logger that should be used by this manager. // If none is set, it defaults to log.Log global logger. logger logr.Logger - // leaderElectionCancel is used to cancel the leader election. It is distinct from internalStopper, - // because for safety reasons we need to os.Exit() when we lose the leader election, meaning that - // it must be deferred until after gracefulShutdown is done. - leaderElectionCancel context.CancelFunc - // leaderElectionStopped is an internal channel used to signal the stopping procedure that the // LeaderElection.Run(...) function has returned and the shutdown can proceed. leaderElectionStopped chan struct{} - // stop procedure engaged. In other words, we should not add anything else to the manager - stopProcedureEngaged bool + // leaderElectionCancel is used to cancel the leader election. It is distinct from internalStopper, + // because for safety reasons we need to os.Exit() when we lose the leader election, meaning that + // it must be deferred until after gracefulShutdown is done. + leaderElectionCancel context.CancelFunc // elected is closed when this manager becomes the leader of a group of // managers, either because it won a leader election or because no leader // election was configured. elected chan struct{} - caches []hasCache - - // port is the port that the webhook server serves at. - port int - // host is the hostname that the webhook server binds to. - host string - // CertDir is the directory that contains the server key and certificate. - // if not set, webhook server would look up the server key and certificate in - // {TempDir}/k8s-webhook-server/serving-certs - certDir string - - webhookServer *webhook.Server + webhookServer webhook.Server // webhookServerOnce will be called in GetWebhookServer() to optionally initialize // webhookServer if unset, and Add() it to controllerManager. webhookServerOnce sync.Once + // leaderElectionID is the name of the resource that leader election + // will use for holding the leader lock. + leaderElectionID string // leaseDuration is the duration that non-leader candidates will // wait to force acquire leadership. leaseDuration time.Duration @@ -160,10 +142,6 @@ type controllerManager struct { // between tries of actions. retryPeriod time.Duration - // waitForRunnable is holding the number of runnables currently running so that - // we can wait for them to exit before quitting the manager - waitForRunnable sync.WaitGroup - // gracefulShutdownTimeout is the duration given to runnable to stop // before the manager actually returns on stop. gracefulShutdownTimeout time.Duration @@ -192,84 +170,39 @@ type hasCache interface { // Add sets dependencies on i, and adds it to the list of Runnables to start. func (cm *controllerManager) Add(r Runnable) error { - cm.mu.Lock() - defer cm.mu.Unlock() - if cm.stopProcedureEngaged { - return errors.New("can't accept new runnable as stop procedure is already engaged") - } - - // Set dependencies on the object - if err := cm.SetFields(r); err != nil { - return err - } - - var shouldStart bool - - // Add the runnable to the leader election or the non-leaderelection list - if leRunnable, ok := r.(LeaderElectionRunnable); ok && !leRunnable.NeedLeaderElection() { - shouldStart = cm.started - cm.nonLeaderElectionRunnables = append(cm.nonLeaderElectionRunnables, r) - } else if hasCache, ok := r.(hasCache); ok { - cm.caches = append(cm.caches, hasCache) - } else { - shouldStart = cm.startedLeader - cm.leaderElectionRunnables = append(cm.leaderElectionRunnables, r) - } - - if shouldStart { - // If already started, start the controller - cm.startRunnable(r) - } + cm.Lock() + defer cm.Unlock() + return cm.add(r) +} - return nil +func (cm *controllerManager) add(r Runnable) error { + return cm.runnables.Add(r) } -// Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10. -func (cm *controllerManager) SetFields(i interface{}) error { - if err := cm.cluster.SetFields(i); err != nil { - return err - } - if _, err := inject.InjectorInto(cm.SetFields, i); err != nil { - return err +// AddMetricsServerExtraHandler adds extra handler served on path to the http server that serves metrics. +func (cm *controllerManager) AddMetricsServerExtraHandler(path string, handler http.Handler) error { + cm.Lock() + defer cm.Unlock() + if cm.started { + return fmt.Errorf("unable to add new metrics handler because metrics endpoint has already been created") } - if _, err := inject.StopChannelInto(cm.internalProceduresStop, i); err != nil { - return err + if cm.metricsServer == nil { + cm.GetLogger().Info("warn: metrics server is currently disabled, registering extra handler will be ignored", "path", path) + return nil } - if _, err := inject.LoggerInto(cm.logger, i); err != nil { + if err := cm.metricsServer.AddExtraHandler(path, handler); err != nil { return err } - - return nil -} - -// AddMetricsExtraHandler adds extra handler served on path to the http server that serves metrics. -func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Handler) error { - if path == defaultMetricsEndpoint { - return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint) - } - - cm.mu.Lock() - defer cm.mu.Unlock() - - if _, found := cm.metricsExtraHandlers[path]; found { - return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path) - } - - cm.metricsExtraHandlers[path] = handler cm.logger.V(2).Info("Registering metrics http server extra handler", "path", path) return nil } // AddHealthzCheck allows you to add Healthz checker. func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) error { - cm.mu.Lock() - defer cm.mu.Unlock() - - if cm.stopProcedureEngaged { - return errors.New("can't accept new healthCheck as stop procedure is already engaged") - } + cm.Lock() + defer cm.Unlock() - if cm.healthzStarted { + if cm.started { return fmt.Errorf("unable to add new checker because healthz endpoint has already been created") } @@ -283,15 +216,11 @@ func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) // AddReadyzCheck allows you to add Readyz checker. func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker) error { - cm.mu.Lock() - defer cm.mu.Unlock() - - if cm.stopProcedureEngaged { - return errors.New("can't accept new ready check as stop procedure is already engaged") - } + cm.Lock() + defer cm.Unlock() - if cm.healthzStarted { - return fmt.Errorf("unable to add new checker because readyz endpoint has already been created") + if cm.started { + return fmt.Errorf("unable to add new checker because healthz endpoint has already been created") } if cm.readyzHandler == nil { @@ -302,6 +231,10 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker) return nil } +func (cm *controllerManager) GetHTTPClient() *http.Client { + return cm.cluster.GetHTTPClient() +} + func (cm *controllerManager) GetConfig() *rest.Config { return cm.cluster.GetConfig() } @@ -334,17 +267,13 @@ func (cm *controllerManager) GetAPIReader() client.Reader { return cm.cluster.GetAPIReader() } -func (cm *controllerManager) GetWebhookServer() *webhook.Server { +func (cm *controllerManager) GetWebhookServer() webhook.Server { cm.webhookServerOnce.Do(func() { if cm.webhookServer == nil { - cm.webhookServer = &webhook.Server{ - Port: cm.port, - Host: cm.host, - CertDir: cm.certDir, - } + panic("webhook should not be nil") } if err := cm.Add(cm.webhookServer); err != nil { - panic("unable to add webhook server to the controller manager") + panic(fmt.Sprintf("unable to add webhook server to the controller manager: %s", err)) } }) return cm.webhookServer @@ -354,89 +283,83 @@ func (cm *controllerManager) GetLogger() logr.Logger { return cm.logger } -func (cm *controllerManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { - return cm.controllerOptions +func (cm *controllerManager) GetControllerOptions() config.Controller { + return cm.controllerConfig } -func (cm *controllerManager) serveMetrics() { - handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ - ErrorHandling: promhttp.HTTPErrorOnError, - }) - // TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics +func (cm *controllerManager) addHealthProbeServer() error { mux := http.NewServeMux() - mux.Handle(defaultMetricsEndpoint, handler) - - func() { - cm.mu.Lock() - defer cm.mu.Unlock() - - for path, extraHandler := range cm.metricsExtraHandlers { - mux.Handle(path, extraHandler) - } - }() + srv := httpserver.New(mux) - server := http.Server{ - Handler: mux, + if cm.readyzHandler != nil { + mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) + // Append '/' suffix to handle subpaths + mux.Handle(cm.readinessEndpointName+"/", http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) } - // Run the server - cm.startRunnable(RunnableFunc(func(_ context.Context) error { - cm.logger.Info("starting metrics server", "path", defaultMetricsEndpoint) - if err := server.Serve(cm.metricsListener); err != nil && err != http.ErrServerClosed { - return err - } - return nil - })) - - // Shutdown the server when stop is closed - <-cm.internalProceduresStop - if err := server.Shutdown(cm.shutdownCtx); err != nil { - cm.errChan <- err + if cm.healthzHandler != nil { + mux.Handle(cm.livenessEndpointName, http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) + // Append '/' suffix to handle subpaths + mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) } + + return cm.add(&Server{ + Name: "health probe", + Server: srv, + Listener: cm.healthProbeListener, + }) } -func (cm *controllerManager) serveHealthProbes() { +func (cm *controllerManager) addPprofServer() error { mux := http.NewServeMux() - server := http.Server{ - Handler: mux, - } + srv := httpserver.New(mux) + + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + return cm.add(&Server{ + Name: "pprof", + Server: srv, + Listener: cm.pprofListener, + }) +} - func() { - cm.mu.Lock() - defer cm.mu.Unlock() +// Start starts the manager and waits indefinitely. +// There is only two ways to have start return: +// An error has occurred during in one of the internal operations, +// such as leader election, cache start, webhooks, and so on. +// Or, the context is cancelled. +func (cm *controllerManager) Start(ctx context.Context) (err error) { + cm.Lock() + if cm.started { + cm.Unlock() + return errors.New("manager already started") + } + cm.started = true - if cm.readyzHandler != nil { - mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) - // Append '/' suffix to handle subpaths - mux.Handle(cm.readinessEndpointName+"/", http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) - } - if cm.healthzHandler != nil { - mux.Handle(cm.livenessEndpointName, http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) - // Append '/' suffix to handle subpaths - mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) + var ready bool + defer func() { + // Only unlock the manager if we haven't reached + // the internal readiness condition. + if !ready { + cm.Unlock() } - - // Run server - cm.startRunnable(RunnableFunc(func(_ context.Context) error { - if err := server.Serve(cm.healthProbeListener); err != nil && err != http.ErrServerClosed { - return err - } - return nil - })) - cm.healthzStarted = true }() - // Shutdown the server when stop is closed - <-cm.internalProceduresStop - if err := server.Shutdown(cm.shutdownCtx); err != nil { - cm.errChan <- err - } -} + // Initialize the internal context. + cm.internalCtx, cm.internalCancel = context.WithCancel(ctx) -func (cm *controllerManager) Start(ctx context.Context) (err error) { - if err := cm.Add(cm.cluster); err != nil { - return fmt.Errorf("failed to add cluster to runnables: %w", err) + // Leader elector must be created before defer that contains engageStopProcedure function + // https://github.com/kubernetes-sigs/controller-runtime/issues/2873 + var leaderElector *leaderelection.LeaderElector + if cm.resourceLock != nil { + leaderElector, err = cm.initLeaderElector() + if err != nil { + return fmt.Errorf("failed during initialization leader election process: %w", err) + } } - cm.internalCtx, cm.internalCancel = context.WithCancel(ctx) // This chan indicates that stop is complete, in other words all runnables have returned or timeout on stop request stopComplete := make(chan struct{}) @@ -457,40 +380,97 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { } }() - // initialize this here so that we reset the signal channel state on every start - // Everything that might write into this channel must be started in a new goroutine, - // because otherwise we might block this routine trying to write into the full channel - // and will not be able to enter the deferred cm.engageStopProcedure() which drains - // it. - cm.errChan = make(chan error) + // Add the cluster runnable. + if err := cm.add(cm.cluster); err != nil { + return fmt.Errorf("failed to add cluster to runnables: %w", err) + } // Metrics should be served whether the controller is leader or not. // (If we don't serve metrics for non-leaders, prometheus will still scrape - // the pod but will get a connection refused) - if cm.metricsListener != nil { - go cm.serveMetrics() + // the pod but will get a connection refused). + if cm.metricsServer != nil { + // Note: We are adding the metrics server directly to HTTPServers here as matching on the + // metricsserver.Server interface in cm.runnables.Add would be very brittle. + if err := cm.runnables.HTTPServers.Add(cm.metricsServer, nil); err != nil { + return fmt.Errorf("failed to add metrics server: %w", err) + } } - // Serve health probes + // Serve health probes. if cm.healthProbeListener != nil { - go cm.serveHealthProbes() + if err := cm.addHealthProbeServer(); err != nil { + return fmt.Errorf("failed to add health probe server: %w", err) + } } - go cm.startNonLeaderElectionRunnables() + // Add pprof server + if cm.pprofListener != nil { + if err := cm.addPprofServer(); err != nil { + return fmt.Errorf("failed to add pprof server: %w", err) + } + } - go func() { - if cm.resourceLock != nil { - err := cm.startLeaderElection() - if err != nil { - cm.errChan <- err - } + // First start any HTTP servers, which includes health probes, metrics and profiling if enabled. + // + // WARNING: HTTPServers includes the health probes, which MUST start before any cache is populated, otherwise + // it would block conversion webhooks to be ready for serving which make the cache never get ready. + logCtx := logr.NewContext(cm.internalCtx, cm.logger) + if err := cm.runnables.HTTPServers.Start(logCtx); err != nil { + return fmt.Errorf("failed to start HTTP servers: %w", err) + } + + // Start any webhook servers, which includes conversion, validation, and defaulting + // webhooks that are registered. + // + // WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition + // between conversion webhooks and the cache sync (usually initial list) which causes the webhooks + // to never start because no cache can be populated. + if err := cm.runnables.Webhooks.Start(cm.internalCtx); err != nil { + return fmt.Errorf("failed to start webhooks: %w", err) + } + + // Start and wait for caches. + if err := cm.runnables.Caches.Start(cm.internalCtx); err != nil { + return fmt.Errorf("failed to start caches: %w", err) + } + + // Start the non-leaderelection Runnables after the cache has synced. + if err := cm.runnables.Others.Start(cm.internalCtx); err != nil { + return fmt.Errorf("failed to start other runnables: %w", err) + } + + // Start WarmupRunnables and wait for warmup to complete. + if err := cm.runnables.Warmup.Start(cm.internalCtx); err != nil { + return fmt.Errorf("failed to start warmup runnables: %w", err) + } + + // Start the leader election and all required runnables. + { + // Create a context that inherits all keys from the parent context + // but can be cancelled independently for leader election management + baseCtx := context.WithoutCancel(ctx) + leaderCtx, cancel := context.WithCancel(baseCtx) + cm.leaderElectionCancel = cancel + if leaderElector != nil { + // Start the leader elector process + go func() { + leaderElector.Run(leaderCtx) + <-leaderCtx.Done() + close(cm.leaderElectionStopped) + }() } else { - // Treat not having leader election enabled the same as being elected. - cm.startLeaderElectionRunnables() - close(cm.elected) + go func() { + // Treat not having leader election enabled the same as being elected. + if err := cm.startLeaderElectionRunnables(); err != nil { + cm.errChan <- err + } + close(cm.elected) + }() } - }() + } + ready = true + cm.Unlock() select { case <-ctx.Done(): // We are done @@ -504,27 +484,39 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { // engageStopProcedure signals all runnables to stop, reads potential errors // from the errChan and waits for them to end. It must not be called more than once. func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) error { - // Populate the shutdown context. + if !atomic.CompareAndSwapInt64(cm.stopProcedureEngaged, 0, 1) { + return errors.New("stop procedure already engaged") + } + + // Populate the shutdown context, this operation MUST be done before + // closing the internalProceduresStop channel. + // + // The shutdown context immediately expires if the gracefulShutdownTimeout is not set. var shutdownCancel context.CancelFunc - if cm.gracefulShutdownTimeout > 0 { - cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout) - } else { + if cm.gracefulShutdownTimeout < 0 { + // We want to wait forever for the runnables to stop. cm.shutdownCtx, shutdownCancel = context.WithCancel(context.Background()) + } else { + cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout) } defer shutdownCancel() - // Cancel the internal stop channel and wait for the procedures to stop and complete. - close(cm.internalProceduresStop) - cm.internalCancel() - // Start draining the errors before acquiring the lock to make sure we don't deadlock // if something that has the lock is blocked on trying to write into the unbuffered // channel after something else already wrote into it. + var closeOnce sync.Once go func() { for { + // Closing in the for loop is required to avoid race conditions between + // the closure of all internal procedures and making sure to have a reader off the error channel. + closeOnce.Do(func() { + // Cancel the internal stop channel and wait for the procedures to stop and complete. + close(cm.internalProceduresStop) + cm.internalCancel() + }) select { - case err, ok := <-cm.errChan: - if ok { + case err := <-cm.errChan: + if !errors.Is(err, context.Canceled) { cm.logger.Error(err, "error received after stop sequence was engaged") } case <-stopComplete: @@ -532,26 +524,14 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e } } }() - if cm.gracefulShutdownTimeout == 0 { - return nil - } - cm.mu.Lock() - defer cm.mu.Unlock() - cm.stopProcedureEngaged = true - // we want to close this after the other runnables stop, because we don't + // We want to close this after the other runnables stop, because we don't // want things like leader election to try and emit events on a closed // channel defer cm.recorderProvider.Stop(cm.shutdownCtx) - return cm.waitForRunnableToEnd(shutdownCancel) -} - -// waitForRunnableToEnd blocks until all runnables ended or the -// tearDownTimeout was reached. In the latter case, an error is returned. -func (cm *controllerManager) waitForRunnableToEnd(shutdownCancel context.CancelFunc) (retErr error) { - // Cancel leader election only after we waited. It will os.Exit() the app for safety. defer func() { - if retErr == nil && cm.leaderElectionCancel != nil { + // Cancel leader election only after we waited. It will os.Exit() the app for safety. + if cm.resourceLock != nil { // After asking the context to be cancelled, make sure // we wait for the leader stopped channel to be closed, otherwise // we might encounter race conditions between this code @@ -562,139 +542,102 @@ func (cm *controllerManager) waitForRunnableToEnd(shutdownCancel context.CancelF }() go func() { - cm.waitForRunnable.Wait() + go func() { + // Stop the warmup runnables in a separate goroutine to avoid blocking. + // It is important to stop the warmup runnables in parallel with the other runnables + // since we cannot assume ordering of whether or not one of the warmup runnables or one + // of the other runnables is holding a lock. + // Cancelling the wrong runnable (one that is not holding the lock) will cause the + // shutdown sequence to block indefinitely as it will wait for the runnable that is + // holding the lock to finish. + cm.logger.Info("Stopping and waiting for warmup runnables") + cm.runnables.Warmup.StopAndWait(cm.shutdownCtx) + }() + + // First stop the non-leader election runnables. + cm.logger.Info("Stopping and waiting for non leader election runnables") + cm.runnables.Others.StopAndWait(cm.shutdownCtx) + + // Stop all the leader election runnables, which includes reconcilers. + cm.logger.Info("Stopping and waiting for leader election runnables") + // Prevent leader election when shutting down a non-elected manager + cm.runnables.LeaderElection.startOnce.Do(func() {}) + cm.runnables.LeaderElection.StopAndWait(cm.shutdownCtx) + + // Stop the caches before the leader election runnables, this is an important + // step to make sure that we don't race with the reconcilers by receiving more events + // from the API servers and enqueueing them. + cm.logger.Info("Stopping and waiting for caches") + cm.runnables.Caches.StopAndWait(cm.shutdownCtx) + + // Webhooks and internal HTTP servers should come last, as they might be still serving some requests. + cm.logger.Info("Stopping and waiting for webhooks") + cm.runnables.Webhooks.StopAndWait(cm.shutdownCtx) + + cm.logger.Info("Stopping and waiting for HTTP servers") + cm.runnables.HTTPServers.StopAndWait(cm.shutdownCtx) + + // Proceed to close the manager and overall shutdown context. + cm.logger.Info("Wait completed, proceeding to shutdown the manager") shutdownCancel() }() <-cm.shutdownCtx.Done() - if err := cm.shutdownCtx.Err(); err != nil && err != context.Canceled { - return fmt.Errorf("failed waiting for all runnables to end within grace period of %s: %w", cm.gracefulShutdownTimeout, err) - } - return nil -} - -func (cm *controllerManager) startNonLeaderElectionRunnables() { - cm.mu.Lock() - defer cm.mu.Unlock() - - // First start any webhook servers, which includes conversion, validation, and defaulting - // webhooks that are registered. - // - // WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition - // between conversion webhooks and the cache sync (usually initial list) which causes the webhooks - // to never start because no cache can be populated. - for _, c := range cm.nonLeaderElectionRunnables { - if _, ok := c.(*webhook.Server); ok { - cm.startRunnable(c) - } - } - - // Start and wait for caches. - cm.waitForCache(cm.internalCtx) - - // Start the non-leaderelection Runnables after the cache has synced - for _, c := range cm.nonLeaderElectionRunnables { - if _, ok := c.(*webhook.Server); ok { - continue + if err := cm.shutdownCtx.Err(); err != nil && !errors.Is(err, context.Canceled) { + if errors.Is(err, context.DeadlineExceeded) { + if cm.gracefulShutdownTimeout > 0 { + return fmt.Errorf("failed waiting for all runnables to end within grace period of %s: %w", cm.gracefulShutdownTimeout, err) + } + return nil } - - // Controllers block, but we want to return an error if any have an error starting. - // Write any Start errors to a channel so we can return them - cm.startRunnable(c) - } -} - -func (cm *controllerManager) startLeaderElectionRunnables() { - cm.mu.Lock() - defer cm.mu.Unlock() - - cm.waitForCache(cm.internalCtx) - - // Start the leader election Runnables after the cache has synced - for _, c := range cm.leaderElectionRunnables { - // Controllers block, but we want to return an error if any have an error starting. - // Write any Start errors to a channel so we can return them - cm.startRunnable(c) - } - - cm.startedLeader = true -} - -func (cm *controllerManager) waitForCache(ctx context.Context) { - if cm.started { - return - } - - for _, cache := range cm.caches { - cm.startRunnable(cache) + // For any other error, return the error. + return err } - // Wait for the caches to sync. - // TODO(community): Check the return value and write a test - for _, cache := range cm.caches { - cache.GetCache().WaitForCacheSync(ctx) - } - // TODO: This should be the return value of cm.cache.WaitForCacheSync but we abuse - // cm.started as check if we already started the cache so it must always become true. - // Making sure that the cache doesn't get started twice is needed to not get a "close - // of closed channel" panic - cm.started = true + return nil } -func (cm *controllerManager) startLeaderElection() (err error) { - ctx, cancel := context.WithCancel(context.Background()) - cm.mu.Lock() - cm.leaderElectionCancel = cancel - cm.mu.Unlock() - - if cm.onStoppedLeading == nil { - cm.onStoppedLeading = func() { - // Make sure graceful shutdown is skipped if we lost the leader lock without - // intending to. - cm.gracefulShutdownTimeout = time.Duration(0) - // Most implementations of leader election log.Fatal() here. - // Since Start is wrapped in log.Fatal when called, we can just return - // an error here which will cause the program to exit. - cm.errChan <- errors.New("leader election lost") - } - } - l, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ +func (cm *controllerManager) initLeaderElector() (*leaderelection.LeaderElector, error) { + leaderElector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ Lock: cm.resourceLock, LeaseDuration: cm.leaseDuration, RenewDeadline: cm.renewDeadline, RetryPeriod: cm.retryPeriod, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: func(_ context.Context) { - cm.startLeaderElectionRunnables() + if err := cm.startLeaderElectionRunnables(); err != nil { + cm.errChan <- err + return + } close(cm.elected) }, - OnStoppedLeading: cm.onStoppedLeading, + OnStoppedLeading: func() { + if cm.onStoppedLeading != nil { + cm.onStoppedLeading() + } + // Make sure graceful shutdown is skipped if we lost the leader lock without + // intending to. + cm.gracefulShutdownTimeout = time.Duration(0) + // Most implementations of leader election log.Fatal() here. + // Since Start is wrapped in log.Fatal when called, we can just return + // an error here which will cause the program to exit. + cm.errChan <- errors.New("leader election lost") + }, }, ReleaseOnCancel: cm.leaderElectionReleaseOnCancel, + Name: cm.leaderElectionID, }) if err != nil { - return err + return nil, err } - // Start the leader elector process - go func() { - l.Run(ctx) - <-ctx.Done() - close(cm.leaderElectionStopped) - }() - return nil + return leaderElector, nil } -func (cm *controllerManager) Elected() <-chan struct{} { - return cm.elected +func (cm *controllerManager) startLeaderElectionRunnables() error { + return cm.runnables.LeaderElection.Start(cm.internalCtx) } -func (cm *controllerManager) startRunnable(r Runnable) { - cm.waitForRunnable.Add(1) - go func() { - defer cm.waitForRunnable.Done() - if err := r.Start(cm.internalCtx); err != nil { - cm.errChan <- err - } - }() +func (cm *controllerManager) Elected() <-chan struct{} { + return cm.elected } diff --git a/pkg/manager/internal/integration/api/v1/driver_types.go b/pkg/manager/internal/integration/api/v1/driver_types.go new file mode 100644 index 0000000000..9182ed4cc8 --- /dev/null +++ b/pkg/manager/internal/integration/api/v1/driver_types.go @@ -0,0 +1,93 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// Driver is a test type. +type Driver struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` +} + +// DriverList is a list of Drivers. +type DriverList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Driver `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Driver{}, &DriverList{}) +} + +// DeepCopyInto deep copies into the given Driver. +func (d *Driver) DeepCopyInto(out *Driver) { + *out = *d + out.TypeMeta = d.TypeMeta + d.ObjectMeta.DeepCopyInto(&out.ObjectMeta) +} + +// DeepCopy returns a copy of Driver. +func (d *Driver) DeepCopy() *Driver { + if d == nil { + return nil + } + out := new(Driver) + d.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of Driver as runtime.Object. +func (d *Driver) DeepCopyObject() runtime.Object { + return d.DeepCopy() +} + +// DeepCopyInto deep copies into the given DriverList. +func (in *DriverList) DeepCopyInto(out *DriverList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Driver, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy returns a copy of DriverList. +func (in *DriverList) DeepCopy() *DriverList { + if in == nil { + return nil + } + out := new(DriverList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of DriverList as runtime.Object. +func (in *DriverList) DeepCopyObject() runtime.Object { + return in.DeepCopy() +} + +// Hub marks Driver as a Hub for conversion. +func (*Driver) Hub() {} diff --git a/pkg/config/v1alpha1/register.go b/pkg/manager/internal/integration/api/v1/groupversion_info.go similarity index 76% rename from pkg/config/v1alpha1/register.go rename to pkg/manager/internal/integration/api/v1/groupversion_info.go index 9efdbc0668..3986a6d023 100644 --- a/pkg/config/v1alpha1/register.go +++ b/pkg/manager/internal/integration/api/v1/groupversion_info.go @@ -1,11 +1,11 @@ /* -Copyright 2020 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -14,16 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package v1alpha1 +package v1 import ( "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( // GroupVersion is group version used to register these objects. - GroupVersion = schema.GroupVersion{Group: "controller-runtime.sigs.k8s.io", Version: "v1alpha1"} + GroupVersion = schema.GroupVersion{Group: "crew.example.com", Version: "v1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme. SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} @@ -31,7 +32,3 @@ var ( // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) - -func init() { - SchemeBuilder.Register(&ControllerManagerConfiguration{}) -} diff --git a/pkg/manager/internal/integration/api/v2/driver_types.go b/pkg/manager/internal/integration/api/v2/driver_types.go new file mode 100644 index 0000000000..64012ac749 --- /dev/null +++ b/pkg/manager/internal/integration/api/v2/driver_types.go @@ -0,0 +1,111 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/conversion" + v1 "sigs.k8s.io/controller-runtime/pkg/manager/internal/integration/api/v1" +) + +// Driver is a test type. +type Driver struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` +} + +// DriverList is a list of Drivers. +type DriverList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Driver `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Driver{}, &DriverList{}) +} + +// DeepCopyInto deep copies into the given Driver. +func (d *Driver) DeepCopyInto(out *Driver) { + *out = *d + out.TypeMeta = d.TypeMeta + d.ObjectMeta.DeepCopyInto(&out.ObjectMeta) +} + +// DeepCopy returns a copy of Driver. +func (d *Driver) DeepCopy() *Driver { + if d == nil { + return nil + } + out := new(Driver) + d.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of Driver as runtime.Object. +func (d *Driver) DeepCopyObject() runtime.Object { + return d.DeepCopy() +} + +// DeepCopyInto deep copies into the given DriverList. +func (in *DriverList) DeepCopyInto(out *DriverList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Driver, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy returns a copy of DriverList. +func (in *DriverList) DeepCopy() *DriverList { + if in == nil { + return nil + } + out := new(DriverList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of DriverList as runtime.Object. +func (in *DriverList) DeepCopyObject() runtime.Object { + return in.DeepCopy() +} + +// ConvertTo converts Driver to the Hub version of driver. +func (d *Driver) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1.Driver) + dst.Name = d.Name + dst.Namespace = d.Namespace + dst.UID = d.UID + return nil +} + +// ConvertFrom converts Driver from the Hub version of driver. +func (d *Driver) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1.Driver) + d.Name = src.Name + d.Namespace = src.Namespace + d.UID = src.UID + return nil +} diff --git a/examples/configfile/custom/v1alpha1/types.go b/pkg/manager/internal/integration/api/v2/groupversion_info.go similarity index 51% rename from examples/configfile/custom/v1alpha1/types.go rename to pkg/manager/internal/integration/api/v2/groupversion_info.go index 79e8422c5c..218a742793 100644 --- a/examples/configfile/custom/v1alpha1/types.go +++ b/pkg/manager/internal/integration/api/v2/groupversion_info.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,41 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package v1alpha1 provides the CustomControllerManagerConfiguration used for -// demoing componentconfig -// +kubebuilder:object:generate=true -package v1alpha1 +package v2 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "examples.x-k8s.io", Version: "v1alpha1"} + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "crew.example.com", Version: "v2"} - // SchemeBuilder is used to add go types to the GroupVersionKind scheme + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) - -// +kubebuilder:object:root=true - -// CustomControllerManagerConfiguration is the Schema for the CustomControllerManagerConfigurations API -type CustomControllerManagerConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // ControllerManagerConfigurationSpec returns the contfigurations for controllers - cfg.ControllerManagerConfigurationSpec `json:",inline"` - - ClusterName string `json:"clusterName,omitempty"` -} - -func init() { - SchemeBuilder.Register(&CustomControllerManagerConfiguration{}) -} diff --git a/hack/tools/tools.go b/pkg/manager/internal/integration/manager_suite_test.go similarity index 67% rename from hack/tools/tools.go rename to pkg/manager/internal/integration/manager_suite_test.go index 481a7c6f01..1a5a20d5a5 100644 --- a/hack/tools/tools.go +++ b/pkg/manager/internal/integration/manager_suite_test.go @@ -1,7 +1,5 @@ -// +build tools - /* -Copyright 2019 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,10 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -// This package imports things required by build scripts, to force `go mod` to see them as dependencies -package tools +package integration import ( - _ "github.com/joelanford/go-apidiff" - _ "sigs.k8s.io/controller-tools/cmd/controller-gen" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) + +func TestManager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Manager Integration Suite") +} diff --git a/pkg/manager/internal/integration/manager_test.go b/pkg/manager/internal/integration/manager_test.go new file mode 100644 index 0000000000..c83eead3c1 --- /dev/null +++ b/pkg/manager/internal/integration/manager_test.go @@ -0,0 +1,306 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "net" + "net/http" + "reflect" + "sync/atomic" + "time" + "unsafe" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + crewv1 "sigs.k8s.io/controller-runtime/pkg/manager/internal/integration/api/v1" + crewv2 "sigs.k8s.io/controller-runtime/pkg/manager/internal/integration/api/v2" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/conversion" +) + +var ( + scheme = runtime.NewScheme() + + driverCRD = &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "drivers.crew.example.com", + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: crewv1.GroupVersion.Group, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "drivers", + Singular: "driver", + Kind: "Driver", + }, + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: crewv1.GroupVersion.Version, + Served: true, + // v1 will be the storage version. + // Reconciler and index will use v2 so we can validate the conversion webhook works. + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + }, + }, + }, + { + Name: crewv2.GroupVersion.Version, + Served: true, + Storage: false, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + }, + }, + }, + }, + }, + } + + ctx = ctrl.SetupSignalHandler() +) + +var _ = Describe("manger.Manager Start", func() { + // This test ensure the Manager starts without running into any deadlocks as it can be very tricky + // to start health probes, webhooks, caches (including informers) and reconcilers in the right order. + // + // To verify this we set up a test environment in the following state: + // * Ensure Informer sync requires a functioning conversion webhook (and thus readiness probe) + // * Driver CRD is deployed with v1 as storage version + // * A Driver CR is created and stored in the v1 version + // * Setup manager: + // * Set up health probes + // * Set up a Driver v2 reconciler to verify reconciliation works + // * Set up a conversion webhook which only works if readiness probe succeeds (just like via a Kubernetes service) + // * Add an index on v2 Driver to ensure we start and wait for an informer during cache.Start (as part of manager.Start) + // * Note: cache.Start would fail if the conversion webhook doesn't work (which in turn depends on the readiness probe) + // * Note: Adding the index for v2 ensures the Driver list call during Informer sync goes through conversion. + DescribeTable("should start all components without deadlock", func(warmupEnabled bool) { + // Set up schema. + Expect(clientgoscheme.AddToScheme(scheme)).To(Succeed()) + Expect(apiextensionsv1.AddToScheme(scheme)).To(Succeed()) + Expect(crewv1.AddToScheme(scheme)).To(Succeed()) + Expect(crewv2.AddToScheme(scheme)).To(Succeed()) + + // Set up test environment. + env := &envtest.Environment{ + Scheme: scheme, + CRDInstallOptions: envtest.CRDInstallOptions{ + CRDs: []*apiextensionsv1.CustomResourceDefinition{driverCRD}, + }, + } + // Note: The test env configures a conversion webhook on driverCRD during Start. + cfg, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + defer func() { + Expect(env.Stop()).To(Succeed()) + }() + c, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + + // Create driver CR (which is stored as v1). + driverV1 := &unstructured.Unstructured{} + driverV1.SetGroupVersionKind(crewv1.GroupVersion.WithKind("Driver")) + driverV1.SetName("driver1") + driverV1.SetNamespace(metav1.NamespaceDefault) + Expect(c.Create(ctx, driverV1)).To(Succeed()) + + // Set up Manager. + ctrl.SetLogger(zap.New()) + mgr, err := manager.New(env.Config, manager.Options{ + Scheme: scheme, + HealthProbeBindAddress: ":0", + // Disable metrics to avoid port conflicts. + Metrics: metricsserver.Options{BindAddress: "0"}, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: env.WebhookInstallOptions.LocalServingPort, + Host: env.WebhookInstallOptions.LocalServingHost, + CertDir: env.WebhookInstallOptions.LocalServingCertDir, + }), + }) + Expect(err).NotTo(HaveOccurred()) + + // Configure health probes. + Expect(mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed()) + Expect(mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed()) + + // Set up Driver reconciler (using v2). + driverReconciler := &DriverReconciler{ + Client: mgr.GetClient(), + } + Expect( + ctrl.NewControllerManagedBy(mgr). + For(&crewv2.Driver{}). + Named(fmt.Sprintf("driver_warmup_%t", warmupEnabled)). + WithOptions(controller.Options{EnableWarmup: ptr.To(warmupEnabled)}). + Complete(driverReconciler), + ).To(Succeed()) + + // Set up a conversion webhook. + conversionWebhook := createConversionWebhook(mgr) + mgr.GetWebhookServer().Register("/convert", conversionWebhook) + + // Add an index on Driver (using v2). + // Note: This triggers the creation of an Informer for Driver v2. + Expect(mgr.GetCache().IndexField(ctx, &crewv2.Driver{}, "name", func(object client.Object) []string { + return []string{object.GetName()} + })).To(Succeed()) + + // Start the Manager. + ctx, cancel := context.WithCancel(ctx) + go func() { + defer GinkgoRecover() + Expect(mgr.Start(ctx)).To(Succeed()) + }() + + // Verify manager.Start successfully started health probes, webhooks, caches (including informers) and reconcilers. + // Notes: + // * The cache will only start successfully if the informer for v2 Driver is synced. + // * The informer for v2 Driver will only sync if a list on v2 Driver succeeds (which requires a working conversion webhook) + select { + case <-time.After(30 * time.Second): + // Don't wait forever if the manager doesn't come up. + Fail("Manager didn't start in time") + case <-mgr.Elected(): + } + + // Verify the reconciler reconciles. + Eventually(func(g Gomega) { + g.Expect(atomic.LoadUint64(&driverReconciler.ReconcileCount)).Should(BeNumerically(">", 0)) + }, 10*time.Second).Should(Succeed()) + + // Verify conversion webhook was called. + Expect(atomic.LoadUint64(&conversionWebhook.ConversionCount)).Should(BeNumerically(">", 0)) + + // Verify the conversion webhook works by getting the Driver as v1 and v2. + Expect(c.Get(ctx, client.ObjectKeyFromObject(driverV1), driverV1)).To(Succeed()) + driverV2 := &unstructured.Unstructured{} + driverV2.SetGroupVersionKind(crewv2.GroupVersion.WithKind("Driver")) + driverV2.SetName("driver1") + driverV2.SetNamespace(metav1.NamespaceDefault) + Expect(c.Get(ctx, client.ObjectKeyFromObject(driverV2), driverV2)).To(Succeed()) + + // Shutdown the server + cancel() + }, + Entry("controller warmup enabled", true), + Entry("controller warmup not enabled", false), + ) +}) + +type DriverReconciler struct { + Client client.Client + ReconcileCount uint64 +} + +func (r *DriverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + log := ctrl.LoggerFrom(ctx) + log.Info("Reconciling") + + // Fetch the Driver instance. + cluster := &crewv2.Driver{} + if err := r.Client.Get(ctx, req.NamespacedName, cluster); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + atomic.AddUint64(&r.ReconcileCount, 1) + + return reconcile.Result{}, nil +} + +// ConversionWebhook is just a shim around the conversion handler from +// the webhook package. We use it to simulate the behavior of a conversion +// webhook in a real cluster, i.e. the conversion webhook only works after the +// controller Pod is ready (the readiness probe is up). +type ConversionWebhook struct { + httpClient http.Client + conversionHandler http.Handler + readinessEndpoint string + ConversionCount uint64 +} + +func createConversionWebhook(mgr manager.Manager) *ConversionWebhook { + conversionHandler := conversion.NewWebhookHandler(mgr.GetScheme()) + httpClient := http.Client{ + // Setting a timeout to not get stuck when calling the readiness probe. + Timeout: 5 * time.Second, + } + + // Read the unexported healthProbeListener field of the manager to get the listener address. + // This is a hack but it's better than using a hard-coded port. + v := reflect.ValueOf(mgr).Elem() + field := v.FieldByName("healthProbeListener") + healthProbeListener := *(*net.Listener)(unsafe.Pointer(field.UnsafeAddr())) + readinessEndpoint := fmt.Sprint("http://", healthProbeListener.Addr().String(), "/readyz") + + return &ConversionWebhook{ + httpClient: httpClient, + conversionHandler: conversionHandler, + readinessEndpoint: readinessEndpoint, + } +} + +func (c *ConversionWebhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { + resp, err := c.httpClient.Get(c.readinessEndpoint) + if err != nil { + logf.Log.WithName("conversion-webhook").Error(err, "failed to serve conversion: readiness endpoint is not up") + w.WriteHeader(http.StatusInternalServerError) + return + } + + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + // This simulates the behavior in Kubernetes that conversion webhooks are only served after + // the controller is ready (and thus the Kubernetes service sends requests to the controller). + logf.Log.WithName("conversion-webhook").Info("failed to serve conversion: controller is not ready yet") + w.WriteHeader(http.StatusInternalServerError) + return + } + + atomic.AddUint64(&c.ConversionCount, 1) + c.conversionHandler.ServeHTTP(w, r) +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 2d2733f0a6..e0e94245e7 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -18,31 +18,32 @@ package manager import ( "context" + "errors" "fmt" "net" "net/http" - "reflect" "time" "github.com/go-logr/logr" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" + "k8s.io/utils/ptr" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -53,8 +54,7 @@ type Manager interface { cluster.Cluster // Add will set requested dependencies on the component, and cause the component to be - // started when Start is called. Add will inject any dependencies for which the argument - // implements the inject interface - e.g. inject.Client. + // started when Start is called. // Depending on if a Runnable implements LeaderElectionRunnable interface, a Runnable can be run in either // non-leaderelection mode (always running) or leader election mode (managed by leader election if enabled). Add(Runnable) error @@ -64,12 +64,14 @@ type Manager interface { // election was configured. Elected() <-chan struct{} - // AddMetricsExtraHandler adds an extra handler served on path to the http server that serves metrics. - // Might be useful to register some diagnostic endpoints e.g. pprof. Note that these endpoints meant to be - // sensitive and shouldn't be exposed publicly. - // If the simple path -> handler mapping offered here is not enough, a new http server/listener should be added as - // Runnable to the manager via Add method. - AddMetricsExtraHandler(path string, handler http.Handler) error + // AddMetricsServerExtraHandler adds an extra handler served on path to the http server that serves metrics. + // Might be useful to register some diagnostic endpoints e.g. pprof. + // + // Note that these endpoints are meant to be sensitive and shouldn't be exposed publicly. + // + // If the simple path -> handler mapping offered here is not enough, + // a new http server/listener should be added as Runnable to the manager via Add method. + AddMetricsServerExtraHandler(path string, handler http.Handler) error // AddHealthzCheck allows you to add Healthz checker AddHealthzCheck(name string, check healthz.Checker) error @@ -86,51 +88,58 @@ type Manager interface { Start(ctx context.Context) error // GetWebhookServer returns a webhook.Server - GetWebhookServer() *webhook.Server + GetWebhookServer() webhook.Server // GetLogger returns this manager's logger. GetLogger() logr.Logger // GetControllerOptions returns controller global configuration options. - GetControllerOptions() v1alpha1.ControllerConfigurationSpec + GetControllerOptions() config.Controller } // Options are the arguments for creating a new Manager. type Options struct { - // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources + // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources. // Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better - // idea to pass your own scheme in. See the documentation in pkg/scheme for more information. + // to pass your own scheme in. See the documentation in pkg/scheme for more information. + // + // If set, the Scheme will be used to create the default Client and Cache. Scheme *runtime.Scheme - // MapperProvider provides the rest mapper used to map go types to Kubernetes APIs - MapperProvider func(c *rest.Config) (meta.RESTMapper, error) + // MapperProvider provides the rest mapper used to map go types to Kubernetes APIs. + // + // If set, the RESTMapper returned by this function is used to create the RESTMapper + // used by the Client and Cache. + MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) + + // Cache is the cache.Options that will be used to create the default Cache. + // By default, the cache will watch and list requested objects in all namespaces. + Cache cache.Options - // SyncPeriod determines the minimum frequency at which watched resources are - // reconciled. A lower period will correct entropy more quickly, but reduce - // responsiveness to change if there are many watched resources. Change this - // value only if you know what you are doing. Defaults to 10 hours if unset. - // there will a 10 percent jitter between the SyncPeriod of all controllers - // so that all controllers will not send list requests simultaneously. + // NewCache is the function that will create the cache to be used + // by the manager. If not set this will use the default new cache function. // - // This applies to all controllers. + // When using a custom NewCache, the Cache options will be passed to the + // NewCache function. // - // A period sync happens for two reasons: - // 1. To insure against a bug in the controller that causes an object to not - // be requeued, when it otherwise should be requeued. - // 2. To insure against an unknown bug in controller-runtime, or its dependencies, - // that causes an object to not be requeued, when it otherwise should be - // requeued, or to be removed from the queue, when it otherwise should not - // be removed. + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewCache if you know what you are doing. + NewCache cache.NewCacheFunc + + // Client is the client.Options that will be used to create the default Client. + // By default, the client will use the cache for reads and direct calls for writes. + Client client.Options + + // NewClient is the func that creates the client to be used by the manager. + // If not set this will create a Client backed by a Cache for read operations + // and a direct Client for write operations. // - // If you want - // 1. to insure against missed watch events, or - // 2. to poll services that cannot be watched, - // then we recommend that, instead of changing the default period, the - // controller requeue, with a constant duration `t`, whenever the controller - // is "done" with an object, and would otherwise not requeue it, i.e., we - // recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`, - // instead of `reconcile.Result{}`. - SyncPeriod *time.Duration + // When using a custom NewClient, the Client options will be passed to the + // NewClient function. + // + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewClient if you know what you are doing. + NewClient client.NewClientFunc // Logger is the logger that should be used by this manager. // If none is set, it defaults to log.Log global logger. @@ -141,18 +150,36 @@ type Options struct { LeaderElection bool // LeaderElectionResourceLock determines which resource lock to use for leader election, - // defaults to "configmapsleases". Change this value only if you know what you are doing. - // Otherwise, users of your controller might end up with multiple running instances that + // defaults to "leases". Change this value only if you know what you are doing. + // + // If you are using `configmaps`/`endpoints` resource lock and want to migrate to "leases", + // you might do so by migrating to the respective multilock first ("configmapsleases" or "endpointsleases"), + // which will acquire a leader lock on both resources. + // After all your users have migrated to the multilock, you can go ahead and migrate to "leases". + // Please also keep in mind, that users might skip versions of your controller. + // + // Note: before controller-runtime version v0.7, it was set to "configmaps". + // And from v0.7 to v0.11, the default was "configmapsleases", which was + // used to migrate from configmaps to leases. + // Since the default was "configmapsleases" for over a year, spanning five minor releases, + // any actively maintained operators are very likely to have a released version that uses + // "configmapsleases". Therefore defaulting to "leases" should be safe since v0.12. + // + // So, what do you have to do when you are updating your controller-runtime dependency + // from a lower version to v0.12 or newer? + // - If your operator matches at least one of these conditions: + // - the LeaderElectionResourceLock in your operator has already been explicitly set to "leases" + // - the old controller-runtime version is between v0.7.0 and v0.11.x and the + // LeaderElectionResourceLock wasn't set or was set to "leases"/"configmapsleases"/"endpointsleases" + // feel free to update controller-runtime to v0.12 or newer. + // - Otherwise, you may have to take these steps: + // 1. update controller-runtime to v0.12 or newer in your go.mod + // 2. set LeaderElectionResourceLock to "configmapsleases" (or "endpointsleases") + // 3. package your operator and upgrade it in all your clusters + // 4. only if you have finished 3, you can remove the LeaderElectionResourceLock to use the default "leases" + // Otherwise, your operator might end up with multiple running instances that // each acquired leadership through different resource locks during upgrades and thus // act on the same resources concurrently. - // If you want to migrate to the "leases" resource lock, you might do so by migrating to the - // respective multilock first ("configmapsleases" or "endpointsleases"), which will acquire a - // leader lock on both resources. After all your users have migrated to the multilock, you can - // go ahead and migrate to "leases". Please also keep in mind, that users might skip versions - // of your controller. - // - // Note: before controller-runtime version v0.7, the resource lock was set to "configmaps". - // Please keep this in mind, when planning a proper migration path for your controller. LeaderElectionResourceLock string // LeaderElectionNamespace determines the namespace in which the leader @@ -174,32 +201,36 @@ type Options struct { // LeaseDuration time first. LeaderElectionReleaseOnCancel bool + // LeaderElectionLabels allows a controller to supplement all leader election api calls with a set of custom labels based on + // the replica attempting to acquire leader status. + LeaderElectionLabels map[string]string + + // LeaderElectionResourceLockInterface allows to provide a custom resourcelock.Interface that was created outside + // of the controller-runtime. If this value is set the options LeaderElectionID, LeaderElectionNamespace, + // LeaderElectionResourceLock, LeaseDuration, RenewDeadline, RetryPeriod and LeaderElectionLeases will be ignored. + // This can be useful if you want to use a locking mechanism that is currently not supported, like a MultiLock across + // two Kubernetes clusters. + LeaderElectionResourceLockInterface resourcelock.Interface + // LeaseDuration is the duration that non-leader candidates will // wait to force acquire leadership. This is measured against time of // last observed ack. Default is 15 seconds. LeaseDuration *time.Duration + // RenewDeadline is the duration that the acting controlplane will retry // refreshing leadership before giving up. Default is 10 seconds. RenewDeadline *time.Duration + // RetryPeriod is the duration the LeaderElector clients should wait // between tries of actions. Default is 2 seconds. RetryPeriod *time.Duration - // Namespace if specified restricts the manager's cache to watch objects in - // the desired namespace Defaults to all namespaces - // - // Note: If a namespace is specified, controllers can still Watch for a - // cluster-scoped resource (e.g Node). For namespaced resources the cache - // will only hold objects from the desired namespace. - Namespace string - - // MetricsBindAddress is the TCP address that the controller should bind to - // for serving prometheus metrics. - // It can be set to "0" to disable the metrics serving. - MetricsBindAddress string + // Metrics are the metricsserver.Options that will be used to create the metricsserver.Server. + Metrics metricsserver.Options // HealthProbeBindAddress is the TCP address that the controller should bind to // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. HealthProbeBindAddress string // Readiness probe endpoint name, defaults to "readyz" @@ -208,43 +239,22 @@ type Options struct { // Liveness probe endpoint name, defaults to "healthz" LivenessEndpointName string - // Port is the port that the webhook server serves at. - // It is used to set webhook.Server.Port if WebhookServer is not set. - Port int - // Host is the hostname that the webhook server binds to. - // It is used to set webhook.Server.Host if WebhookServer is not set. - Host string - - // CertDir is the directory that contains the server key and certificate. - // If not set, webhook server would look up the server key and certificate in - // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate - // must be named tls.key and tls.crt, respectively. - // It is used to set webhook.Server.CertDir if WebhookServer is not set. - CertDir string + // PprofBindAddress is the TCP address that the controller should bind to + // for serving pprof. + // It can be set to "" or "0" to disable the pprof serving. + // Since pprof may contain sensitive information, make sure to protect it + // before exposing it to public. + PprofBindAddress string // WebhookServer is an externally configured webhook.Server. By default, - // a Manager will create a default server using Port, Host, and CertDir; - // if this is set, the Manager will use this server instead. - WebhookServer *webhook.Server - - // Functions to all for a user to customize the values that will be injected. - - // NewCache is the function that will create the cache to be used - // by the manager. If not set this will use the default new cache function. - NewCache cache.NewCacheFunc - - // NewClient is the func that creates the client to be used by the manager. - // If not set this will create the default DelegatingClient that will - // use the cache for reads and the client for writes. - NewClient cluster.NewClientFunc - - // ClientDisableCacheFor tells the client that, if any cache is used, to bypass it - // for the given objects. - ClientDisableCacheFor []client.Object + // a Manager will create a server via webhook.NewServer with default settings. + // If this is set, the Manager will use this server instead. + WebhookServer webhook.Server - // DryRunClient specifies whether the client should be configured to enforce - // dryRun mode. - DryRunClient bool + // BaseContext is the function that provides Context values to Runnables + // managed by the Manager. If a BaseContext function isn't provided, Runnables + // will receive a new Background Context instead. + BaseContext BaseContextFunc // EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API // Use this to customize the event correlator and spam filter @@ -262,7 +272,7 @@ type Options struct { // Controller contains global configuration options for controllers // registered within this manager. // +optional - Controller v1alpha1.ControllerConfigurationSpec + Controller config.Controller // makeBroadcaster allows deferring the creation of the broadcaster to // avoid leaking goroutines if we never call Start on this manager. It also @@ -271,12 +281,17 @@ type Options struct { makeBroadcaster intrec.EventBroadcasterProducer // Dependency injection for testing - newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) + newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) - newMetricsListener func(addr string) (net.Listener, error) + newMetricsServer func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) newHealthProbeListener func(addr string) (net.Listener, error) + newPprofListener func(addr string) (net.Listener, error) } +// BaseContextFunc is a function used to provide a base Context to Runnables +// managed by a Manager. +type BaseContextFunc func() context.Context + // Runnable allows a component to be started. // It's very important that Start blocks until // it's done running. @@ -304,8 +319,23 @@ type LeaderElectionRunnable interface { NeedLeaderElection() bool } +// warmupRunnable knows if a Runnable requires warmup. A warmup runnable is a runnable +// that should be run when the manager is started but before it becomes leader. +// Note: Implementing this interface is only useful when LeaderElection can be enabled, as the +// behavior when leaderelection is not enabled is to run LeaderElectionRunnables immediately. +type warmupRunnable interface { + // Warmup will be called when the manager is started but before it becomes leader. + Warmup(context.Context) error +} + // New returns a new Manager for creating Controllers. +// Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf" +// will be used for all built-in resources of Kubernetes, and "application/json" is for other types +// including all CRD resources. func New(config *rest.Config, options Options) (Manager, error) { + if config == nil { + return nil, errors.New("must specify Config") + } // Set default values for options fields options = setOptionsDefaults(options) @@ -313,51 +343,80 @@ func New(config *rest.Config, options Options) (Manager, error) { clusterOptions.Scheme = options.Scheme clusterOptions.MapperProvider = options.MapperProvider clusterOptions.Logger = options.Logger - clusterOptions.SyncPeriod = options.SyncPeriod - clusterOptions.Namespace = options.Namespace clusterOptions.NewCache = options.NewCache clusterOptions.NewClient = options.NewClient - clusterOptions.ClientDisableCacheFor = options.ClientDisableCacheFor - clusterOptions.DryRunClient = options.DryRunClient + clusterOptions.Cache = options.Cache + clusterOptions.Client = options.Client clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck }) if err != nil { return nil, err } + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + // Create the recorder provider to inject event recorders for the components. // TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific // to the particular controller that it's being injected into, rather than a generic one like is here. - recorderProvider, err := options.newRecorderProvider(config, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster) + recorderProvider, err := options.newRecorderProvider(config, cluster.GetHTTPClient(), cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster) if err != nil { return nil, err } // Create the resource lock to enable leader election) - leaderConfig := options.LeaderElectionConfig - if leaderConfig == nil { + var leaderConfig *rest.Config + var leaderRecorderProvider *intrec.Provider + + if options.LeaderElectionConfig == nil { leaderConfig = rest.CopyConfig(config) + leaderRecorderProvider = recorderProvider + } else { + leaderConfig = rest.CopyConfig(options.LeaderElectionConfig) + scheme := cluster.GetScheme() + err := corev1.AddToScheme(scheme) + if err != nil { + return nil, err + } + err = coordinationv1.AddToScheme(scheme) + if err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(options.LeaderElectionConfig) + if err != nil { + return nil, err + } + leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, httpClient, scheme, options.Logger.WithName("events"), options.makeBroadcaster) + if err != nil { + return nil, err + } } - resourceLock, err := options.newResourceLock(leaderConfig, recorderProvider, leaderelection.Options{ - LeaderElection: options.LeaderElection, - LeaderElectionResourceLock: options.LeaderElectionResourceLock, - LeaderElectionID: options.LeaderElectionID, - LeaderElectionNamespace: options.LeaderElectionNamespace, - }) - if err != nil { - return nil, err + + var resourceLock resourcelock.Interface + if options.LeaderElectionResourceLockInterface != nil && options.LeaderElection { + resourceLock = options.LeaderElectionResourceLockInterface + } else { + resourceLock, err = options.newResourceLock(leaderConfig, leaderRecorderProvider, leaderelection.Options{ + LeaderElection: options.LeaderElection, + LeaderElectionResourceLock: options.LeaderElectionResourceLock, + LeaderElectionID: options.LeaderElectionID, + LeaderElectionNamespace: options.LeaderElectionNamespace, + RenewDeadline: *options.RenewDeadline, + LeaderLabels: options.LeaderElectionLabels, + }) + if err != nil { + return nil, err + } } - // Create the metrics listener. This will throw an error if the metrics bind - // address is invalid or already in use. - metricsListener, err := options.newMetricsListener(options.MetricsBindAddress) + // Create the metrics server. + metricsServer, err := options.newMetricsServer(options.Metrics, config, cluster.GetHTTPClient()) if err != nil { return nil, err } - // By default we have no extra endpoints to expose on metrics http server. - metricsExtraHandlers := make(map[string]http.Handler) - // Create health probes listener. This will throw an error if the bind // address is invalid or already in use. healthProbeListener, err := options.newHealthProbeListener(options.HealthProbeBindAddress) @@ -365,25 +424,35 @@ func New(config *rest.Config, options Options) (Manager, error) { return nil, err } + // Create pprof listener. This will throw an error if the bind + // address is invalid or already in use. + pprofListener, err := options.newPprofListener(options.PprofBindAddress) + if err != nil { + return nil, fmt.Errorf("failed to new pprof listener: %w", err) + } + + errChan := make(chan error, 1) + runnables := newRunnables(options.BaseContext, errChan).withLogger(options.Logger) return &controllerManager{ + stopProcedureEngaged: ptr.To(int64(0)), cluster: cluster, + runnables: runnables, + errChan: errChan, recorderProvider: recorderProvider, resourceLock: resourceLock, - metricsListener: metricsListener, - metricsExtraHandlers: metricsExtraHandlers, - controllerOptions: options.Controller, + metricsServer: metricsServer, + controllerConfig: options.Controller, logger: options.Logger, elected: make(chan struct{}), - port: options.Port, - host: options.Host, - certDir: options.CertDir, webhookServer: options.WebhookServer, + leaderElectionID: options.LeaderElectionID, leaseDuration: *options.LeaseDuration, renewDeadline: *options.RenewDeadline, retryPeriod: *options.RetryPeriod, healthProbeListener: healthProbeListener, readinessEndpointName: options.ReadinessEndpointName, livenessEndpointName: options.LivenessEndpointName, + pprofListener: pprofListener, gracefulShutdownTimeout: *options.GracefulShutdownTimeout, internalProceduresStop: make(chan struct{}), leaderElectionStopped: make(chan struct{}), @@ -391,127 +460,38 @@ func New(config *rest.Config, options Options) (Manager, error) { }, nil } -// AndFrom will use a supplied type and convert to Options -// any options already set on Options will be ignored, this is used to allow -// cli flags to override anything specified in the config file. -func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, error) { - if inj, wantsScheme := loader.(inject.Scheme); wantsScheme { - err := inj.InjectScheme(o.Scheme) - if err != nil { - return o, err - } - } - - newObj, err := loader.Complete() - if err != nil { - return o, err - } - - o = o.setLeaderElectionConfig(newObj) - - if o.SyncPeriod == nil && newObj.SyncPeriod != nil { - o.SyncPeriod = &newObj.SyncPeriod.Duration - } - - if o.Namespace == "" && newObj.CacheNamespace != "" { - o.Namespace = newObj.CacheNamespace - } - - if o.MetricsBindAddress == "" && newObj.Metrics.BindAddress != "" { - o.MetricsBindAddress = newObj.Metrics.BindAddress - } - - if o.HealthProbeBindAddress == "" && newObj.Health.HealthProbeBindAddress != "" { - o.HealthProbeBindAddress = newObj.Health.HealthProbeBindAddress - } - - if o.ReadinessEndpointName == "" && newObj.Health.ReadinessEndpointName != "" { - o.ReadinessEndpointName = newObj.Health.ReadinessEndpointName - } - - if o.LivenessEndpointName == "" && newObj.Health.LivenessEndpointName != "" { - o.LivenessEndpointName = newObj.Health.LivenessEndpointName - } - - if o.Port == 0 && newObj.Webhook.Port != nil { - o.Port = *newObj.Webhook.Port - } - - if o.Host == "" && newObj.Webhook.Host != "" { - o.Host = newObj.Webhook.Host - } - - if o.CertDir == "" && newObj.Webhook.CertDir != "" { - o.CertDir = newObj.Webhook.CertDir - } - - if newObj.Controller != nil { - if o.Controller.CacheSyncTimeout == nil && newObj.Controller.CacheSyncTimeout != nil { - o.Controller.CacheSyncTimeout = newObj.Controller.CacheSyncTimeout - } - - if len(o.Controller.GroupKindConcurrency) == 0 && len(newObj.Controller.GroupKindConcurrency) > 0 { - o.Controller.GroupKindConcurrency = newObj.Controller.GroupKindConcurrency - } +// defaultHealthProbeListener creates the default health probes listener bound to the given address. +func defaultHealthProbeListener(addr string) (net.Listener, error) { + if addr == "" || addr == "0" { + return nil, nil } - return o, nil -} - -// AndFromOrDie will use options.AndFrom() and will panic if there are errors. -func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Options { - o, err := o.AndFrom(loader) + ln, err := net.Listen("tcp", addr) if err != nil { - panic(fmt.Sprintf("could not parse config file: %v", err)) - } - return o -} - -func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options { - if !o.LeaderElection && obj.LeaderElection.LeaderElect != nil { - o.LeaderElection = *obj.LeaderElection.LeaderElect - } - - if o.LeaderElectionResourceLock == "" && obj.LeaderElection.ResourceLock != "" { - o.LeaderElectionResourceLock = obj.LeaderElection.ResourceLock - } - - if o.LeaderElectionNamespace == "" && obj.LeaderElection.ResourceNamespace != "" { - o.LeaderElectionNamespace = obj.LeaderElection.ResourceNamespace - } - - if o.LeaderElectionID == "" && obj.LeaderElection.ResourceName != "" { - o.LeaderElectionID = obj.LeaderElection.ResourceName - } - - if o.LeaseDuration == nil && !reflect.DeepEqual(obj.LeaderElection.LeaseDuration, metav1.Duration{}) { - o.LeaseDuration = &obj.LeaderElection.LeaseDuration.Duration - } - - if o.RenewDeadline == nil && !reflect.DeepEqual(obj.LeaderElection.RenewDeadline, metav1.Duration{}) { - o.RenewDeadline = &obj.LeaderElection.RenewDeadline.Duration + return nil, fmt.Errorf("error listening on %s: %w", addr, err) } - - if o.RetryPeriod == nil && !reflect.DeepEqual(obj.LeaderElection.RetryPeriod, metav1.Duration{}) { - o.RetryPeriod = &obj.LeaderElection.RetryPeriod.Duration - } - - return o + return ln, nil } -// defaultHealthProbeListener creates the default health probes listener bound to the given address. -func defaultHealthProbeListener(addr string) (net.Listener, error) { +// defaultPprofListener creates the default pprof listener bound to the given address. +func defaultPprofListener(addr string) (net.Listener, error) { if addr == "" || addr == "0" { return nil, nil } ln, err := net.Listen("tcp", addr) if err != nil { - return nil, fmt.Errorf("error listening on %s: %v", addr, err) + return nil, fmt.Errorf("error listening on %s: %w", addr, err) } return ln, nil } +// defaultBaseContext is used as the BaseContext value in Options if one +// has not already been set. +func defaultBaseContext() context.Context { + return context.Background() +} + // setOptionsDefaults set default values for Options fields. func setOptionsDefaults(options Options) Options { // Allow newResourceLock to be mocked @@ -538,8 +518,8 @@ func setOptionsDefaults(options Options) Options { } } - if options.newMetricsListener == nil { - options.newMetricsListener = metrics.NewListener + if options.newMetricsServer == nil { + options.newMetricsServer = metricsserver.NewServer } leaseDuration, renewDeadline, retryPeriod := defaultLeaseDuration, defaultRenewDeadline, defaultRetryPeriod if options.LeaseDuration == nil { @@ -566,14 +546,30 @@ func setOptionsDefaults(options Options) Options { options.newHealthProbeListener = defaultHealthProbeListener } + if options.newPprofListener == nil { + options.newPprofListener = defaultPprofListener + } + if options.GracefulShutdownTimeout == nil { gracefulShutdownTimeout := defaultGracefulShutdownPeriod options.GracefulShutdownTimeout = &gracefulShutdownTimeout } - if options.Logger == nil { + if options.Logger.GetSink() == nil { options.Logger = log.Log } + if options.Controller.Logger.GetSink() == nil { + options.Controller.Logger = options.Logger + } + + if options.BaseContext == nil { + options.BaseContext = defaultBaseContext + } + + if options.WebhookServer == nil { + options.WebhookServer = webhook.NewServer(webhook.Options{}) + } + return options } diff --git a/pkg/manager/manager_suite_test.go b/pkg/manager/manager_suite_test.go index 3b975a9049..7fbf7184ac 100644 --- a/pkg/manager/manager_suite_test.go +++ b/pkg/manager/manager_suite_test.go @@ -21,21 +21,19 @@ import ( "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Manager Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Manager Suite") } var testenv *envtest.Environment @@ -71,12 +69,12 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) // Prevent the metrics listener being created - metrics.DefaultBindAddress = "0" -}, 60) + metricsserver.DefaultBindAddress = "0" +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) // Put the DefaultBindAddress back - metrics.DefaultBindAddress = ":8080" + metricsserver.DefaultBindAddress = ":8080" }) diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index b697751799..4363d62f59 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -20,7 +20,7 @@ import ( "context" "errors" "fmt" - "io/ioutil" + "io" "net" "net/http" "path" @@ -29,30 +29,28 @@ import ( "time" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + "github.com/go-logr/logr/funcr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" "go.uber.org/goleak" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" - configv1alpha1 "k8s.io/component-base/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/metrics" - "sigs.k8s.io/controller-runtime/pkg/reconcile" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -68,7 +66,7 @@ var _ = Describe("manger.Manager", func() { It("should return an error if it can't create a RestMapper", func() { expected := fmt.Errorf("expected error: RestMapper") m, err := New(cfg, Options{ - MapperProvider: func(c *rest.Config) (meta.RESTMapper, error) { return nil, expected }, + MapperProvider: func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { return nil, expected }, }) Expect(m).To(BeNil()) Expect(err).To(Equal(expected)) @@ -77,7 +75,7 @@ var _ = Describe("manger.Manager", func() { It("should return an error it can't create a client.Client", func() { m, err := New(cfg, Options{ - NewClient: func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { return nil, errors.New("expected error") }, }) @@ -99,7 +97,7 @@ var _ = Describe("manger.Manager", func() { It("should create a client defined in by the new client function", func() { m, err := New(cfg, Options{ - NewClient: func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { return nil, nil }, }) @@ -110,7 +108,7 @@ var _ = Describe("manger.Manager", func() { It("should return an error it can't create a recorder.Provider", func() { m, err := New(cfg, Options{ - newRecorderProvider: func(_ *rest.Config, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { + newRecorderProvider: func(_ *rest.Config, _ *http.Client, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { return nil, fmt.Errorf("expected error") }, }) @@ -119,170 +117,63 @@ var _ = Describe("manger.Manager", func() { Expect(err.Error()).To(ContainSubstring("expected error")) }) - It("should be able to load Options from cfg.ControllerManagerConfiguration type", func() { - duration := metav1.Duration{Duration: 48 * time.Hour} - port := int(6090) - leaderElect := false - - ccfg := &v1alpha1.ControllerManagerConfiguration{ - ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ - SyncPeriod: &duration, - LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ - LeaderElect: &leaderElect, - ResourceLock: "leases", - ResourceNamespace: "default", - ResourceName: "ctrl-lease", - LeaseDuration: duration, - RenewDeadline: duration, - RetryPeriod: duration, - }, - CacheNamespace: "default", - Metrics: v1alpha1.ControllerMetrics{ - BindAddress: ":6000", - }, - Health: v1alpha1.ControllerHealth{ - HealthProbeBindAddress: "6060", - ReadinessEndpointName: "/readyz", - LivenessEndpointName: "/livez", - }, - Webhook: v1alpha1.ControllerWebhook{ - Port: &port, - Host: "localhost", - CertDir: "/certs", - }, - }, - } - - m, err := Options{}.AndFrom(&fakeDeferredLoader{ccfg}) - Expect(err).To(BeNil()) - - Expect(*m.SyncPeriod).To(Equal(duration.Duration)) - Expect(m.LeaderElection).To(Equal(leaderElect)) - Expect(m.LeaderElectionResourceLock).To(Equal("leases")) - Expect(m.LeaderElectionNamespace).To(Equal("default")) - Expect(m.LeaderElectionID).To(Equal("ctrl-lease")) - Expect(m.LeaseDuration.String()).To(Equal(duration.Duration.String())) - Expect(m.RenewDeadline.String()).To(Equal(duration.Duration.String())) - Expect(m.RetryPeriod.String()).To(Equal(duration.Duration.String())) - Expect(m.Namespace).To(Equal("default")) - Expect(m.MetricsBindAddress).To(Equal(":6000")) - Expect(m.HealthProbeBindAddress).To(Equal("6060")) - Expect(m.ReadinessEndpointName).To(Equal("/readyz")) - Expect(m.LivenessEndpointName).To(Equal("/livez")) - Expect(m.Port).To(Equal(port)) - Expect(m.Host).To(Equal("localhost")) - Expect(m.CertDir).To(Equal("/certs")) - }) - - It("should be able to keep Options when cfg.ControllerManagerConfiguration set", func() { - optDuration := time.Duration(2) - duration := metav1.Duration{Duration: 48 * time.Hour} - port := int(6090) - leaderElect := false - - ccfg := &v1alpha1.ControllerManagerConfiguration{ - ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ - SyncPeriod: &duration, - LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ - LeaderElect: &leaderElect, - ResourceLock: "leases", - ResourceNamespace: "default", - ResourceName: "ctrl-lease", - LeaseDuration: duration, - RenewDeadline: duration, - RetryPeriod: duration, - }, - CacheNamespace: "default", - Metrics: v1alpha1.ControllerMetrics{ - BindAddress: ":6000", - }, - Health: v1alpha1.ControllerHealth{ - HealthProbeBindAddress: "6060", - ReadinessEndpointName: "/readyz", - LivenessEndpointName: "/livez", - }, - Webhook: v1alpha1.ControllerWebhook{ - Port: &port, - Host: "localhost", - CertDir: "/certs", - }, - }, - } - - m, err := Options{ - SyncPeriod: &optDuration, - LeaderElection: true, - LeaderElectionResourceLock: "configmaps", - LeaderElectionNamespace: "ctrl", - LeaderElectionID: "ctrl-configmap", - LeaseDuration: &optDuration, - RenewDeadline: &optDuration, - RetryPeriod: &optDuration, - Namespace: "ctrl", - MetricsBindAddress: ":7000", - HealthProbeBindAddress: "5000", - ReadinessEndpointName: "/readiness", - LivenessEndpointName: "/liveness", - Port: 8080, - Host: "example.com", - CertDir: "/pki", - }.AndFrom(&fakeDeferredLoader{ccfg}) - Expect(err).To(BeNil()) - - Expect(m.SyncPeriod.String()).To(Equal(optDuration.String())) - Expect(m.LeaderElection).To(Equal(true)) - Expect(m.LeaderElectionResourceLock).To(Equal("configmaps")) - Expect(m.LeaderElectionNamespace).To(Equal("ctrl")) - Expect(m.LeaderElectionID).To(Equal("ctrl-configmap")) - Expect(m.LeaseDuration.String()).To(Equal(optDuration.String())) - Expect(m.RenewDeadline.String()).To(Equal(optDuration.String())) - Expect(m.RetryPeriod.String()).To(Equal(optDuration.String())) - Expect(m.Namespace).To(Equal("ctrl")) - Expect(m.MetricsBindAddress).To(Equal(":7000")) - Expect(m.HealthProbeBindAddress).To(Equal("5000")) - Expect(m.ReadinessEndpointName).To(Equal("/readiness")) - Expect(m.LivenessEndpointName).To(Equal("/liveness")) - Expect(m.Port).To(Equal(8080)) - Expect(m.Host).To(Equal("example.com")) - Expect(m.CertDir).To(Equal("/pki")) - }) - It("should lazily initialize a webhook server if needed", func() { By("creating a manager with options") - m, err := New(cfg, Options{Port: 9440, Host: "foo.com"}) + m, err := New(cfg, Options{WebhookServer: webhook.NewServer(webhook.Options{Port: 9440, Host: "foo.com"})}) Expect(err).NotTo(HaveOccurred()) Expect(m).NotTo(BeNil()) By("checking options are passed to the webhook server") svr := m.GetWebhookServer() Expect(svr).NotTo(BeNil()) - Expect(svr.Port).To(Equal(9440)) - Expect(svr.Host).To(Equal("foo.com")) + Expect(svr.(*webhook.DefaultServer).Options.Port).To(Equal(9440)) + Expect(svr.(*webhook.DefaultServer).Options.Host).To(Equal("foo.com")) }) It("should not initialize a webhook server if Options.WebhookServer is set", func() { By("creating a manager with options") - m, err := New(cfg, Options{Port: 9441, WebhookServer: &webhook.Server{Port: 9440}}) + srv := webhook.NewServer(webhook.Options{Port: 9440}) + m, err := New(cfg, Options{WebhookServer: srv}) Expect(err).NotTo(HaveOccurred()) Expect(m).NotTo(BeNil()) By("checking the server contains the Port set on the webhook server and not passed to Options") svr := m.GetWebhookServer() Expect(svr).NotTo(BeNil()) - Expect(svr.Port).To(Equal(9440)) + Expect(svr).To(Equal(srv)) + Expect(svr.(*webhook.DefaultServer).Options.Port).To(Equal(9440)) + }) + + It("should allow passing a custom webhook.Server implementation", func() { + type customWebhook struct { + webhook.Server + } + m, err := New(cfg, Options{WebhookServer: customWebhook{}}) + Expect(err).NotTo(HaveOccurred()) + Expect(m).NotTo(BeNil()) + + svr := m.GetWebhookServer() + Expect(svr).NotTo(BeNil()) + + _, isCustomWebhook := svr.(customWebhook) + Expect(isCustomWebhook).To(BeTrue()) }) Context("with leader election enabled", func() { - It("should only cancel the leader election after all runnables are done", func() { + It("should only cancel the leader election after all runnables are done", func(specCtx SpecContext) { m, err := New(cfg, Options{ LeaderElection: true, LeaderElectionNamespace: "default", LeaderElectionID: "test-leader-election-id-2", HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", }) - Expect(err).To(BeNil()) - + Expect(err).ToNot(HaveOccurred()) + gvkcorev1 := schema.GroupVersionKind{Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, Kind: "ConfigMap"} + gvkcoordinationv1 := schema.GroupVersionKind{Group: coordinationv1.SchemeGroupVersion.Group, Version: coordinationv1.SchemeGroupVersion.Version, Kind: "Lease"} + Expect(m.GetScheme().Recognizes(gvkcorev1)).To(BeTrue()) + Expect(m.GetScheme().Recognizes(gvkcoordinationv1)).To(BeTrue()) runnableDone := make(chan struct{}) slowRunnable := RunnableFunc(func(ctx context.Context) error { <-ctx.Done() @@ -290,7 +181,7 @@ var _ = Describe("manger.Manager", func() { close(runnableDone) return nil }) - Expect(m.Add(slowRunnable)).To(BeNil()) + Expect(m.Add(slowRunnable)).To(Succeed()) cm := m.(*controllerManager) cm.gracefulShutdownTimeout = time.Second @@ -299,14 +190,14 @@ var _ = Describe("manger.Manager", func() { close(leaderElectionDone) } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) mgrDone := make(chan struct{}) go func() { defer GinkgoRecover() - Expect(m.Start(ctx)).To(BeNil()) + Expect(m.Start(ctx)).To(Succeed()) close(mgrDone) }() - <-cm.elected + <-cm.Elected() cancel() select { case <-leaderElectionDone: @@ -318,24 +209,23 @@ var _ = Describe("manger.Manager", func() { <-mgrDone }) - It("should disable gracefulShutdown when stopping to lead", func() { + It("should disable gracefulShutdown when stopping to lead", func(ctx SpecContext) { m, err := New(cfg, Options{ LeaderElection: true, LeaderElectionNamespace: "default", LeaderElectionID: "test-leader-election-id-3", HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", }) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() mgrDone := make(chan struct{}) go func() { defer GinkgoRecover() err := m.Start(ctx) - Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(Equal("leader election lost")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("leader election lost")) close(mgrDone) }() cm := m.(*controllerManager) @@ -346,7 +236,106 @@ var _ = Describe("manger.Manager", func() { Expect(cm.gracefulShutdownTimeout.Nanoseconds()).To(Equal(int64(0))) }) - It("should default ID to controller-runtime if ID is not set", func() { + + It("should prevent leader election when shutting down a non-elected manager", func(specCtx SpecContext) { + var rl resourcelock.Interface + m1, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-id", + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + var err error + rl, err = leaderelection.NewResourceLock(config, recorderProvider, options) + return rl, err + }, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m1).ToNot(BeNil()) + Expect(rl.Describe()).To(Equal("default/test-leader-election-id")) + + m1cm, ok := m1.(*controllerManager) + Expect(ok).To(BeTrue()) + m1cm.onStoppedLeading = func() {} + + m2, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-id", + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + var err error + rl, err = leaderelection.NewResourceLock(config, recorderProvider, options) + return rl, err + }, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m2).ToNot(BeNil()) + Expect(rl.Describe()).To(Equal("default/test-leader-election-id")) + + m1done := make(chan struct{}) + Expect(m1.Add(RunnableFunc(func(ctx context.Context) error { + defer GinkgoRecover() + close(m1done) + return nil + }))).To(Succeed()) + + go func() { + defer GinkgoRecover() + Expect(m1.Elected()).ShouldNot(BeClosed()) + Expect(m1.Start(specCtx)).NotTo(HaveOccurred()) + }() + <-m1.Elected() + <-m1done + + electionRunnable := &needElection{make(chan struct{})} + + Expect(m2.Add(electionRunnable)).To(Succeed()) + + ctx2, cancel2 := context.WithCancel(specCtx) + m2done := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(m2.Start(ctx2)).NotTo(HaveOccurred()) + close(m2done) + }() + Consistently(m2.Elected()).ShouldNot(Receive()) + + go func() { + defer GinkgoRecover() + Consistently(electionRunnable.ch).ShouldNot(Receive()) + }() + cancel2() + <-m2done + }) + + It("should default RenewDeadline for leader election config", func() { + var rl resourcelock.Interface + m1, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-id", + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + if options.RenewDeadline != 10*time.Second { + return nil, fmt.Errorf("expected RenewDeadline to be 10s, got %v", options.RenewDeadline) + } + var err error + rl, err = leaderelection.NewResourceLock(config, recorderProvider, options) + return rl, err + }, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m1).ToNot(BeNil()) + }) + + It("should default ID to controller-runtime if ID is not set", func(specCtx SpecContext) { var rl resourcelock.Interface m1, err := New(cfg, Options{ LeaderElection: true, @@ -358,7 +347,8 @@ var _ = Describe("manger.Manager", func() { return rl, err }, HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", }) Expect(err).ToNot(HaveOccurred()) Expect(m1).ToNot(BeNil()) @@ -378,7 +368,8 @@ var _ = Describe("manger.Manager", func() { return rl, err }, HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", }) Expect(err).ToNot(HaveOccurred()) Expect(m2).ToNot(BeNil()) @@ -395,14 +386,12 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx1, cancel1 := context.WithCancel(context.Background()) - defer cancel1() go func() { defer GinkgoRecover() Expect(m1.Elected()).ShouldNot(BeClosed()) - Expect(m1.Start(ctx1)).NotTo(HaveOccurred()) - Expect(m1.Elected()).Should(BeClosed()) + Expect(m1.Start(specCtx)).NotTo(HaveOccurred()) }() + <-m1.Elected() <-c1 c2 := make(chan struct{}) @@ -412,7 +401,7 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx2, cancel := context.WithCancel(context.Background()) + ctx2, cancel := context.WithCancel(specCtx) m2done := make(chan struct{}) go func() { defer GinkgoRecover() @@ -435,6 +424,7 @@ var _ = Describe("manger.Manager", func() { Expect(m).To(BeNil()) Expect(err).To(MatchError(ContainSubstring("expected error"))) }) + It("should return an error if namespace not set and not running in cluster", func() { m, err := New(cfg, Options{LeaderElection: true, LeaderElectionID: "controller-runtime"}) Expect(m).To(BeNil()) @@ -445,18 +435,14 @@ var _ = Describe("manger.Manager", func() { // We must keep this default until we are sure all controller-runtime users have upgraded from the original default // ConfigMap lock to a controller-runtime version that has this new default. Many users of controller-runtime skip // versions, so we should be extremely conservative here. - It("should default to ConfigMapsLeasesResourceLock", func() { + It("should default to LeasesResourceLock", func() { m, err := New(cfg, Options{LeaderElection: true, LeaderElectionID: "controller-runtime", LeaderElectionNamespace: "my-ns"}) Expect(m).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) cm, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - multilock, isMultiLock := cm.resourceLock.(*resourcelock.MultiLock) - Expect(isMultiLock).To(BeTrue()) - _, primaryIsConfigMapLock := multilock.Primary.(*resourcelock.ConfigMapLock) - Expect(primaryIsConfigMapLock).To(BeTrue()) - _, secondaryIsLeaseLock := multilock.Secondary.(*resourcelock.LeaseLock) - Expect(secondaryIsLeaseLock).To(BeTrue()) + _, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock) + Expect(isLeaseLock).To(BeTrue()) }) It("should use the specified ResourceLock", func() { @@ -473,7 +459,7 @@ var _ = Describe("manger.Manager", func() { _, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock) Expect(isLeaseLock).To(BeTrue()) }) - It("should release lease if ElectionReleaseOnCancel is true", func() { + It("should release lease if ElectionReleaseOnCancel is true", func(specCtx SpecContext) { var rl resourcelock.Interface m, err := New(cfg, Options{ LeaderElection: true, @@ -487,9 +473,9 @@ var _ = Describe("manger.Manager", func() { return rl, err }, }) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) doneCh := make(chan struct{}) go func() { defer GinkgoRecover() @@ -500,48 +486,151 @@ var _ = Describe("manger.Manager", func() { cancel() <-doneCh - ctx, cancel = context.WithCancel(context.Background()) - defer cancel() - record, _, err := rl.Get(ctx) - Expect(err).To(BeNil()) + record, _, err := rl.Get(specCtx) + Expect(err).ToNot(HaveOccurred()) Expect(record.HolderIdentity).To(BeEmpty()) }) + It("should set the leaselocks's label field when LeaderElectionLabels is set", func() { + labels := map[string]string{"my-key": "my-val"} + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaderElectionID: "controller-runtime", + LeaderElectionNamespace: "default", + LeaderElectionLabels: labels, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m).ToNot(BeNil()) + cm, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + ll, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock) + Expect(isLeaseLock).To(BeTrue()) + val, exists := ll.Labels["my-key"] + Expect(exists).To(BeTrue()) + Expect(val).To(Equal("my-val")) + }) + When("using a custom LeaderElectionResourceLockInterface", func() { + It("should use the custom LeaderElectionResourceLockInterface", func() { + rl, err := fakeleaderelection.NewResourceLock(nil, nil, leaderelection.Options{}) + Expect(err).NotTo(HaveOccurred()) + + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionResourceLockInterface: rl, + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + return nil, fmt.Errorf("this should not be called") + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + cm, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + Expect(cm.resourceLock).To(Equal(rl)) + }) + }) }) - It("should create a listener for the metrics if a valid address is provided", func() { - var listener net.Listener + It("should create a metrics server if a valid address is provided", func(specCtx SpecContext) { + var srv metricsserver.Server m, err := New(cfg, Options{ - MetricsBindAddress: ":0", - newMetricsListener: func(addr string) (net.Listener, error) { + Metrics: metricsserver.Options{BindAddress: ":0"}, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { var err error - listener, err = metrics.NewListener(addr) - return listener, err + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(srv).ToNot(BeNil()) + + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + ctx, cancel := context.WithTimeout(specCtx, 5*time.Second) + Expect(srv.Start(ctx)).To(Succeed()) + cancel() + }) + + It("should create a metrics server if a valid address is provided and secure serving is enabled", func(specCtx SpecContext) { + var srv metricsserver.Server + m, err := New(cfg, Options{ + Metrics: metricsserver.Options{BindAddress: ":0", SecureServing: true}, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { + var err error + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(srv).ToNot(BeNil()) + + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + ctx, cancel := context.WithTimeout(specCtx, 5*time.Second) + Expect(srv.Start(ctx)).To(Succeed()) + cancel() + }) + + It("should be able to create a manager with a cache that fails on missing informer", func() { + m, err := New(cfg, Options{ + Cache: cache.Options{ + ReaderFailOnMissingInformer: true, }, }) Expect(m).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - Expect(listener).ToNot(BeNil()) - Expect(listener.Close()).ToNot(HaveOccurred()) }) - It("should return an error if the metrics bind address is already in use", func() { - ln, err := metrics.NewListener(":0") + It("should return an error if the metrics bind address is already in use", func(ctx SpecContext) { + ln, err := net.Listen("tcp", ":0") Expect(err).ShouldNot(HaveOccurred()) - var listener net.Listener + var srv metricsserver.Server m, err := New(cfg, Options{ - MetricsBindAddress: ln.Addr().String(), - newMetricsListener: func(addr string) (net.Listener, error) { + Metrics: metricsserver.Options{ + BindAddress: ln.Addr().String(), + }, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { var err error - listener, err = metrics.NewListener(addr) - return listener, err + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err }, }) - Expect(m).To(BeNil()) - Expect(err).To(HaveOccurred()) - Expect(listener).To(BeNil()) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) - Expect(ln.Close()).ToNot(HaveOccurred()) + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + Expect(srv.Start(ctx)).ToNot(Succeed()) + + Expect(ln.Close()).To(Succeed()) + }) + + It("should return an error if the metrics bind address is already in use and secure serving enabled", func(ctx SpecContext) { + ln, err := net.Listen("tcp", ":0") + Expect(err).ShouldNot(HaveOccurred()) + + var srv metricsserver.Server + m, err := New(cfg, Options{ + Metrics: metricsserver.Options{ + BindAddress: ln.Addr().String(), + SecureServing: true, + }, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { + var err error + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + Expect(srv.Start(ctx)).ToNot(Succeed()) + + Expect(ln.Close()).To(Succeed()) }) It("should create a listener for the health probes if a valid address is provided", func() { @@ -583,7 +672,7 @@ var _ = Describe("manger.Manager", func() { Describe("Start", func() { var startSuite = func(options Options, callbacks ...func(Manager)) { - It("should Start each Component", func() { + It("should Start each Component", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -603,15 +692,13 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Elected()).ShouldNot(BeClosed()) Expect(m.Start(ctx)).NotTo(HaveOccurred()) - Expect(m.Elected()).Should(BeClosed()) }() + <-m.Elected() wgRunnableStarted.Wait() }) @@ -634,18 +721,18 @@ var _ = Describe("manger.Manager", func() { Expect(m.GetConfig()).To(Equal(originalCfg)) }) - It("should stop when context is cancelled", func() { + It("should stop when context is cancelled", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { cb(m) } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }) - It("should return an error if it can't start the cache", func() { + It("should return an error if it can't start the cache", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -653,14 +740,14 @@ var _ = Describe("manger.Manager", func() { } mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - mgr.caches = []hasCache{&cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}}} + Expect(mgr.Add( + &cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}}, + )).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() Expect(m.Start(ctx)).To(MatchError(ContainSubstring("expected error"))) }) - It("should start the cache before starting anything else", func() { + It("should start the cache before starting anything else", func(ctx SpecContext) { fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { return fakeCache, nil @@ -672,17 +759,16 @@ var _ = Describe("manger.Manager", func() { } runnableWasStarted := make(chan struct{}) - Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + runnable := RunnableFunc(func(ctx context.Context) error { defer GinkgoRecover() if !fakeCache.wasSynced { return errors.New("runnable got started before cache was synced") } close(runnableWasStarted) return nil - }))).To(Succeed()) + }) + Expect(m.Add(runnable)).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).ToNot(HaveOccurred()) @@ -691,7 +777,7 @@ var _ = Describe("manger.Manager", func() { <-runnableWasStarted }) - It("should start additional clusters before anything else", func() { + It("should start additional clusters before anything else", func(ctx SpecContext) { fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { return fakeCache, nil @@ -724,8 +810,6 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).ToNot(HaveOccurred()) @@ -734,7 +818,7 @@ var _ = Describe("manger.Manager", func() { <-runnableWasStarted }) - It("should return an error if any Components fail to Start", func() { + It("should return an error if any Components fail to Start", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -757,15 +841,53 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - defer GinkgoRecover() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() err = m.Start(ctx) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("expected error")) }) - It("should wait for runnables to stop", func() { + It("should start caches added after Manager has started", func(ctx SpecContext) { + fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} + options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { + return fakeCache, nil + } + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + + runnableWasStarted := make(chan struct{}) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + defer GinkgoRecover() + if !fakeCache.wasSynced { + return errors.New("WaitForCacheSyncCalled wasn't called before Runnable got started") + } + close(runnableWasStarted) + return nil + }))).To(Succeed()) + + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).ToNot(HaveOccurred()) + }() + + <-runnableWasStarted + + additionalClusterCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} + fakeCluster := &startClusterAfterManager{informer: additionalClusterCache} + + Expect(err).NotTo(HaveOccurred()) + Expect(m.Add(fakeCluster)).NotTo(HaveOccurred()) + + Eventually(func() bool { + fakeCluster.informer.mu.Lock() + defer fakeCluster.informer.mu.Unlock() + return fakeCluster.informer.wasStarted && fakeCluster.informer.wasSynced + }).Should(BeTrue()) + }) + + It("should wait for runnables to stop", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -799,7 +921,7 @@ var _ = Describe("manger.Manager", func() { }))).To(Succeed()) defer GinkgoRecover() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) var wgManagerRunning sync.WaitGroup wgManagerRunning.Add(1) @@ -817,7 +939,7 @@ var _ = Describe("manger.Manager", func() { wgManagerRunning.Wait() }) - It("should return an error if any Components fail to Start and wait for runnables to stop", func() { + It("should return an error if any Components fail to Start and wait for runnables to stop", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -845,13 +967,11 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() Expect(m.Start(ctx)).To(HaveOccurred()) Expect(runnableDoneCount).To(Equal(2)) }) - It("should refuse to add runnable if stop procedure is already engaged", func() { + It("should refuse to add runnable if stop procedure is already engaged", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -868,7 +988,7 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() @@ -881,53 +1001,148 @@ var _ = Describe("manger.Manager", func() { }))).NotTo(Succeed()) }) - It("should return both runnables and stop errors when both error", func() { - m, err := New(cfg, options) - Expect(err).NotTo(HaveOccurred()) - for _, cb := range callbacks { - cb(m) + It("should not return runnables context.Canceled errors", func(specCtx SpecContext) { + Expect(options.Logger).To(BeZero(), "this test overrides Logger") + + var log struct { + sync.Mutex + messages []string } - m.(*controllerManager).gracefulShutdownTimeout = 1 * time.Nanosecond - Expect(m.Add(RunnableFunc(func(context.Context) error { - return runnableError{} - }))) - testDone := make(chan struct{}) - defer close(testDone) - Expect(m.Add(RunnableFunc(func(ctx context.Context) error { - <-ctx.Done() - timer := time.NewTimer(30 * time.Second) - defer timer.Stop() - select { - case <-testDone: - return nil - case <-timer.C: - return nil - } - }))) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - err = m.Start(ctx) - Expect(err).ToNot(BeNil()) - eMsg := "[not feeling like that, failed waiting for all runnables to end within grace period of 1ns: context deadline exceeded]" - Expect(err.Error()).To(Equal(eMsg)) - Expect(errors.Is(err, context.DeadlineExceeded)).To(BeTrue()) - Expect(errors.Is(err, runnableError{})).To(BeTrue()) - }) + options.Logger = funcr.NewJSON(func(object string) { + log.Lock() + log.messages = append(log.messages, object) + log.Unlock() + }, funcr.Options{}) - It("should return only stop errors if runnables dont error", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { cb(m) } - m.(*controllerManager).gracefulShutdownTimeout = 1 * time.Nanosecond + + // Runnables may return ctx.Err() as shown in some [context.Context] examples. + started := make(chan struct{}) Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + close(started) <-ctx.Done() - return nil - }))) - testDone := make(chan struct{}) - defer close(testDone) - Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + return ctx.Err() + }))).To(Succeed()) + + stopped := make(chan error) + ctx, cancel := context.WithCancel(specCtx) + go func() { + stopped <- m.Start(ctx) + }() + + // Wait for runnables to start, signal the manager, and wait for it to return. + <-started + cancel() + Expect(<-stopped).To(Succeed()) + + // The leader election goroutine emits one more log message after Start() returns. + // Take the lock here to avoid a race between it writing to log.messages and the + // following read from log.messages. + if options.LeaderElection { + log.Lock() + defer log.Unlock() + } + + Expect(log.messages).To(Not(ContainElement( + ContainSubstring(context.Canceled.Error()), + ))) + }) + + It("should default controller logger from manager logger", func(specCtx SpecContext) { + var lock sync.Mutex + var messages []string + options.Logger = funcr.NewJSON(func(object string) { + lock.Lock() + messages = append(messages, object) + lock.Unlock() + }, funcr.Options{}) + options.LeaderElection = false + + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + + started := make(chan struct{}) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + close(started) + return nil + }))).To(Succeed()) + + stopped := make(chan error) + ctx, cancel := context.WithCancel(specCtx) + go func() { + stopped <- m.Start(ctx) + }() + + // Wait for runnables to start as a proxy for the manager being fully started. + <-started + cancel() + Expect(<-stopped).To(Succeed()) + + msg := "controller log message" + m.GetControllerOptions().Logger.Info(msg) + + Eventually(func(g Gomega) { + lock.Lock() + defer lock.Unlock() + + g.Expect(messages).To(ContainElement( + ContainSubstring(msg), + )) + }).Should(Succeed()) + }) + + It("should return both runnables and stop errors when both error", func(ctx SpecContext) { + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + m.(*controllerManager).gracefulShutdownTimeout = 1 * time.Nanosecond + Expect(m.Add(RunnableFunc(func(context.Context) error { + return runnableError{} + }))).To(Succeed()) + testDone := make(chan struct{}) + defer close(testDone) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + timer := time.NewTimer(30 * time.Second) + defer timer.Stop() + select { + case <-testDone: + return nil + case <-timer.C: + return nil + } + }))).To(Succeed()) + err = m.Start(ctx) + Expect(err).To(HaveOccurred()) + eMsg := "[not feeling like that, failed waiting for all runnables to end within grace period of 1ns: context deadline exceeded]" + Expect(err.Error()).To(Equal(eMsg)) + Expect(errors.Is(err, context.DeadlineExceeded)).To(BeTrue()) + Expect(errors.Is(err, runnableError{})).To(BeTrue()) + }) + + It("should return only stop errors if runnables dont error", func(specCtx SpecContext) { + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + m.(*controllerManager).gracefulShutdownTimeout = 1 * time.Nanosecond + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + return nil + }))).To(Succeed()) + testDone := make(chan struct{}) + defer close(testDone) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { <-ctx.Done() timer := time.NewTimer(30 * time.Second) defer timer.Stop() @@ -937,8 +1152,8 @@ var _ = Describe("manger.Manager", func() { case <-timer.C: return nil } - }))).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + }))).To(Succeed()) + ctx, cancel := context.WithCancel(specCtx) managerStopDone := make(chan struct{}) go func() { err = m.Start(ctx); close(managerStopDone) }() // Use the 'elected' channel to find out if startup was done, otherwise we stop @@ -946,13 +1161,13 @@ var _ = Describe("manger.Manager", func() { <-m.(*controllerManager).elected cancel() <-managerStopDone - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("failed waiting for all runnables to end within grace period of 1ns: context deadline exceeded")) Expect(errors.Is(err, context.DeadlineExceeded)).To(BeTrue()) Expect(errors.Is(err, runnableError{})).ToNot(BeTrue()) }) - It("should return only runnables error if stop doesn't error", func() { + It("should return only runnables error if stop doesn't error", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -960,17 +1175,15 @@ var _ = Describe("manger.Manager", func() { } Expect(m.Add(RunnableFunc(func(context.Context) error { return runnableError{} - }))) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + }))).To(Succeed()) err = m.Start(ctx) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("not feeling like that")) Expect(errors.Is(err, context.DeadlineExceeded)).ToNot(BeTrue()) Expect(errors.Is(err, runnableError{})).To(BeTrue()) }) - It("should not wait for runnables if gracefulShutdownTimeout is 0", func() { + It("should not wait for runnables if gracefulShutdownTimeout is 0", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -986,19 +1199,64 @@ var _ = Describe("manger.Manager", func() { return nil }))).ToNot(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) managerStopDone := make(chan struct{}) go func() { + defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) close(managerStopDone) }() - <-m.(*controllerManager).elected + <-m.Elected() cancel() <-managerStopDone <-runnableStopped }) + It("should wait forever for runnables if gracefulShutdownTimeout is <0 (-1)", func(specCtx SpecContext) { + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + m.(*controllerManager).gracefulShutdownTimeout = time.Duration(-1) + + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(100 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(200 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(500 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(1500 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + + ctx, cancel := context.WithCancel(specCtx) + managerStopDone := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + close(managerStopDone) + }() + <-m.Elected() + cancel() + + beforeDone := time.Now() + <-managerStopDone + Expect(time.Since(beforeDone)).To(BeNumerically(">=", 1500*time.Millisecond)) + }) + } Context("with defaults", func() { @@ -1019,42 +1277,63 @@ var _ = Describe("manger.Manager", func() { cm.onStoppedLeading = func() {} }, ) + + It("should return an error if leader election param incorrect", func(specCtx SpecContext) { + renewDeadline := time.Second * 20 + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionID: "controller-runtime", + LeaderElectionNamespace: "default", + newResourceLock: fakeleaderelection.NewResourceLock, + RenewDeadline: &renewDeadline, + }) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithTimeout(specCtx, time.Second*10) + defer cancel() + err = m.Start(ctx) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, context.DeadlineExceeded)).NotTo(BeTrue()) + }) }) Context("should start serving metrics", func() { - var listener net.Listener + var srv metricsserver.Server + var defaultServer metricsDefaultServer var opts Options BeforeEach(func() { - listener = nil + srv = nil opts = Options{ - newMetricsListener: func(addr string) (net.Listener, error) { + Metrics: metricsserver.Options{ + BindAddress: ":0", + }, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { var err error - listener, err = metrics.NewListener(addr) - return listener, err + srv, err = metricsserver.NewServer(options, config, httpClient) + if srv != nil { + defaultServer = srv.(metricsDefaultServer) + } + return srv, err }, } }) - AfterEach(func() { - if listener != nil { - listener.Close() - } - }) - - It("should stop serving metrics when stop is called", func() { - opts.MetricsBindAddress = ":0" + It("should stop serving metrics when stop is called", func(specCtx SpecContext) { m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) // Check the metrics started - endpoint := fmt.Sprintf("http://%s", listener.Addr().String()) + endpoint := fmt.Sprintf("http://%s/metrics", defaultServer.GetBindAddr()) _, err = http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) @@ -1065,46 +1344,49 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { _, err = http.Get(endpoint) return err - }).ShouldNot(Succeed()) + }, 10*time.Second).ShouldNot(Succeed()) }) - It("should serve metrics endpoint", func() { - opts.MetricsBindAddress = ":0" + It("should serve metrics endpoint", func(ctx SpecContext) { m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - metricsEndpoint := fmt.Sprintf("http://%s/metrics", listener.Addr().String()) + metricsEndpoint := fmt.Sprintf("http://%s/metrics", defaultServer.GetBindAddr()) resp, err := http.Get(metricsEndpoint) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(200)) }) - It("should not serve anything other than metrics endpoint by default", func() { - opts.MetricsBindAddress = ":0" + It("should not serve anything other than metrics endpoint by default", func(ctx SpecContext) { m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - endpoint := fmt.Sprintf("http://%s/should-not-exist", listener.Addr().String()) + endpoint := fmt.Sprintf("http://%s/should-not-exist", defaultServer.GetBindAddr()) resp, err := http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(404)) }) - It("should serve metrics in its registry", func() { + It("should serve metrics in its registry", func(ctx SpecContext) { one := prometheus.NewCounter(prometheus.CounterOpts{ Name: "test_one", Help: "test metric for testing", @@ -1113,23 +1395,25 @@ var _ = Describe("manger.Manager", func() { err := metrics.Registry.Register(one) Expect(err).NotTo(HaveOccurred()) - opts.MetricsBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - metricsEndpoint := fmt.Sprintf("http://%s/metrics", listener.Addr().String()) + metricsEndpoint := fmt.Sprintf("http://%s/metrics", defaultServer.GetBindAddr()) resp, err := http.Get(metricsEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(200)) - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) Expect(err).NotTo(HaveOccurred()) Expect(string(data)).To(ContainSubstring("%s\n%s\n%s\n", `# HELP test_one test metric for testing`, @@ -1142,35 +1426,37 @@ var _ = Describe("manger.Manager", func() { Expect(ok).To(BeTrue()) }) - It("should serve extra endpoints", func() { - opts.MetricsBindAddress = ":0" + It("should serve extra endpoints", func(ctx SpecContext) { + opts.Metrics.ExtraHandlers = map[string]http.Handler{ + "/debug": http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = w.Write([]byte("Some debug info")) + }), + } m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - err = m.AddMetricsExtraHandler("/debug", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - _, _ = w.Write([]byte("Some debug info")) - })) - Expect(err).NotTo(HaveOccurred()) - // Should error when we add another extra endpoint on the already registered path. - err = m.AddMetricsExtraHandler("/debug", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + err = m.AddMetricsServerExtraHandler("/debug", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { _, _ = w.Write([]byte("Another debug info")) })) Expect(err).To(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - endpoint := fmt.Sprintf("http://%s/debug", listener.Addr().String()) + endpoint := fmt.Sprintf("http://%s/debug", defaultServer.GetBindAddr()) resp, err := http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) Expect(err).NotTo(HaveOccurred()) Expect(string(body)).To(Equal("Some debug info")) }) @@ -1198,16 +1484,17 @@ var _ = Describe("manger.Manager", func() { } }) - It("should stop serving health probes when stop is called", func() { + It("should stop serving health probes when stop is called", func(specCtx SpecContext) { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() // Check the health probes started endpoint := fmt.Sprintf("http://%s", listener.Addr().String()) @@ -1221,10 +1508,10 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { _, err = http.Get(endpoint) return err - }).ShouldNot(Succeed()) + }, 10*time.Second).ShouldNot(Succeed()) }) - It("should serve readiness endpoint", func() { + It("should serve readiness endpoint", func(ctx SpecContext) { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1234,24 +1521,25 @@ var _ = Describe("manger.Manager", func() { err = m.AddReadyzCheck(namedCheck, func(_ *http.Request) error { return res }) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() readinessEndpoint := fmt.Sprint("http://", listener.Addr().String(), defaultReadinessEndpoint) // Controller is not ready resp, err := http.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) // Controller is ready res = nil resp, err = http.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check readiness path without trailing slash without redirect @@ -1264,6 +1552,7 @@ var _ = Describe("manger.Manager", func() { } resp, err = httpClient.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check readiness path for individual check @@ -1271,10 +1560,11 @@ var _ = Describe("manger.Manager", func() { res = nil resp, err = http.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) - It("should serve liveness endpoint", func() { + It("should serve liveness endpoint", func(ctx SpecContext) { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1284,24 +1574,25 @@ var _ = Describe("manger.Manager", func() { err = m.AddHealthzCheck(namedCheck, func(_ *http.Request) error { return res }) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() livenessEndpoint := fmt.Sprint("http://", listener.Addr().String(), defaultLivenessEndpoint) // Controller is not ready resp, err := http.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) // Controller is ready res = nil resp, err = http.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check liveness path without trailing slash without redirect @@ -1314,6 +1605,7 @@ var _ = Describe("manger.Manager", func() { } resp, err = httpClient.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check readiness path for individual check @@ -1321,13 +1613,105 @@ var _ = Describe("manger.Manager", func() { res = nil resp, err = http.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + }) + }) + + Context("should start serving pprof", func() { + var listener net.Listener + var opts Options + + BeforeEach(func() { + listener = nil + opts = Options{ + newPprofListener: func(addr string) (net.Listener, error) { + var err error + listener, err = defaultPprofListener(addr) + return listener, err + }, + } + }) + + AfterEach(func() { + if listener != nil { + listener.Close() + } + }) + + It("should stop serving pprof when stop is called", func(specCtx SpecContext) { + opts.PprofBindAddress = ":0" + m, err := New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + ctx, cancel := context.WithCancel(specCtx) + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + + // Check the pprof started + endpoint := fmt.Sprintf("http://%s", listener.Addr().String()) + _, err = http.Get(endpoint) + Expect(err).NotTo(HaveOccurred()) + + // Shutdown the server + cancel() + + // Expect the pprof server to shutdown + Eventually(func() error { + _, err = http.Get(endpoint) + return err + }, 10*time.Second).ShouldNot(Succeed()) + }) + + It("should serve pprof endpoints", func(ctx SpecContext) { + opts.PprofBindAddress = ":0" + m, err := New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + + pprofIndexEndpoint := fmt.Sprintf("http://%s/debug/pprof/", listener.Addr().String()) + resp, err := http.Get(pprofIndexEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + pprofCmdlineEndpoint := fmt.Sprintf("http://%s/debug/pprof/cmdline", listener.Addr().String()) + resp, err = http.Get(pprofCmdlineEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + pprofProfileEndpoint := fmt.Sprintf("http://%s/debug/pprof/profile", listener.Addr().String()) + resp, err = http.Get(pprofProfileEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + pprofSymbolEndpoint := fmt.Sprintf("http://%s/debug/pprof/symbol", listener.Addr().String()) + resp, err = http.Get(pprofSymbolEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + + pprofTraceEndpoint := fmt.Sprintf("http://%s/debug/pprof/trace", listener.Addr().String()) + resp, err = http.Get(pprofTraceEndpoint) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) }) Describe("Add", func() { It("should immediately start the Component if the Manager has already Started another Component", - func() { + func(ctx SpecContext) { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) @@ -1341,18 +1725,15 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() // Wait for the Manager to start Eventually(func() bool { - mgr.mu.Lock() - defer mgr.mu.Unlock() - return mgr.started + return mgr.runnables.Caches.Started() }).Should(BeTrue()) // Add another component after starting @@ -1366,14 +1747,12 @@ var _ = Describe("manger.Manager", func() { <-c2 }) - It("should immediately start the Component if the Manager has already Started", func() { + It("should immediately start the Component if the Manager has already Started", func(ctx SpecContext) { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1381,9 +1760,7 @@ var _ = Describe("manger.Manager", func() { // Wait for the Manager to start Eventually(func() bool { - mgr.mu.Lock() - defer mgr.mu.Unlock() - return mgr.started + return mgr.runnables.Caches.Started() }).Should(BeTrue()) c1 := make(chan struct{}) @@ -1395,115 +1772,35 @@ var _ = Describe("manger.Manager", func() { <-c1 }) - It("should fail if SetFields fails", func() { + It("should fail if attempted to start a second time", func(ctx SpecContext) { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(m.Add(&failRec{})).To(HaveOccurred()) - }) - }) - Describe("SetFields", func() { - It("should inject field values", func() { - m, err := New(cfg, Options{ - NewCache: func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { - return &informertest.FakeInformers{}, nil - }, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Injecting the dependencies") - err = m.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - defer GinkgoRecover() - Expect(scheme).To(Equal(m.GetScheme())) - return nil - }, - config: func(config *rest.Config) error { - defer GinkgoRecover() - Expect(config).To(Equal(m.GetConfig())) - return nil - }, - client: func(client client.Client) error { - defer GinkgoRecover() - Expect(client).To(Equal(m.GetClient())) - return nil - }, - cache: func(c cache.Cache) error { - defer GinkgoRecover() - Expect(c).To(Equal(m.GetCache())) - return nil - }, - stop: func(stop <-chan struct{}) error { - defer GinkgoRecover() - Expect(stop).NotTo(BeNil()) - return nil - }, - f: func(f inject.Func) error { - defer GinkgoRecover() - Expect(f).NotTo(BeNil()) - return nil - }, - log: func(logger logr.Logger) error { - defer GinkgoRecover() - Expect(logger).To(Equal(log.Log)) - return nil - }, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Returning an error if dependency injection fails") - - expected := fmt.Errorf("expected error") - err = m.SetFields(&injectable{ - client: func(client client.Client) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - err = m.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = m.SetFields(&injectable{ - config: func(config *rest.Config) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = m.SetFields(&injectable{ - cache: func(c cache.Cache) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + // Wait for the Manager to start + Eventually(func() bool { + mgr, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + return mgr.runnables.Caches.Started() + }).Should(BeTrue()) - err = m.SetFields(&injectable{ - f: func(c inject.Func) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) + err = m.Start(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("manager already started")) - err = m.SetFields(&injectable{ - stop: func(<-chan struct{}) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) }) }) - It("should not leak goroutines when stopped", func() { + It("should not leak goroutines when stopped", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1513,7 +1810,7 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) }) - It("should not leak goroutines if the default event broadcaster is used & events are emitted", func() { + It("should not leak goroutines if the default event broadcaster is used & events are emitted", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() m, err := New(cfg, Options{ /* implicit: default setting for EventBroadcaster */ }) @@ -1530,15 +1827,17 @@ var _ = Describe("manger.Manager", func() { }))).To(Succeed()) By("starting the manager & waiting till we've sent our event") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) doneCh := make(chan struct{}) go func() { defer GinkgoRecover() defer close(doneCh) Expect(m.Start(ctx)).To(Succeed()) }() + <-m.Elected() + Eventually(func() *corev1.Event { - evts, err := clientset.CoreV1().Events("").Search(m.GetScheme(), &ns) + evts, err := clientset.CoreV1().Events("").SearchWithContext(ctx, m.GetScheme(), &ns) Expect(err).NotTo(HaveOccurred()) for i, evt := range evts.Items { @@ -1559,6 +1858,54 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) }) + It("should not leak goroutines when a runnable returns error slowly after being signaled to stop", func(specCtx SpecContext) { + // This test reproduces the race condition where the manager's Start method + // exits due to context cancellation, leaving no one to drain errChan + + currentGRs := goleak.IgnoreCurrent() + + // Create manager with a very short graceful shutdown timeout to reliablytrigger the race condition + shortGracefulShutdownTimeout := 10 * time.Millisecond + m, err := New(cfg, Options{ + GracefulShutdownTimeout: &shortGracefulShutdownTimeout, + }) + Expect(err).NotTo(HaveOccurred()) + + // Add the slow runnable that will return an error after some delay + for i := 0; i < 3; i++ { + slowRunnable := RunnableFunc(func(c context.Context) error { + <-c.Done() + + // Simulate some work that delays the error from being returned + // Choosing a large delay to reliably trigger the race condition + time.Sleep(100 * time.Millisecond) + + // This simulates the race condition where runnables try to send + // errors after the manager has stopped reading from errChan + return errors.New("slow runnable error") + }) + + Expect(m.Add(slowRunnable)).To(Succeed()) + } + + ctx, cancel := context.WithTimeout(specCtx, 50*time.Millisecond) + defer cancel() + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).To(HaveOccurred()) // We expect error here because the slow runnables will return errors + }() + + // Wait for context to be cancelled + <-ctx.Done() + + // Give time for any leaks to become apparent. This makes sure that we don't false alarm on go routine leaks because runnables are still running. + time.Sleep(300 * time.Millisecond) + + // force-close keep-alive connections + clientTransport.CloseIdleConnections() + Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) + }) + It("should provide a function to get the Config", func() { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1601,95 +1948,77 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) Expect(m.GetAPIReader()).NotTo(BeNil()) }) -}) - -var _ reconcile.Reconciler = &failRec{} -var _ inject.Client = &failRec{} - -type failRec struct{} - -func (*failRec) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, nil -} -func (*failRec) Start(context.Context) error { - return nil -} - -func (*failRec) InjectClient(client.Client) error { - return fmt.Errorf("expected error") -} - -var _ inject.Injector = &injectable{} -var _ inject.Cache = &injectable{} -var _ inject.Client = &injectable{} -var _ inject.Scheme = &injectable{} -var _ inject.Config = &injectable{} -var _ inject.Stoppable = &injectable{} -var _ inject.Logger = &injectable{} - -type injectable struct { - scheme func(scheme *runtime.Scheme) error - client func(client.Client) error - config func(config *rest.Config) error - cache func(cache.Cache) error - f func(inject.Func) error - stop func(<-chan struct{}) error - log func(logger logr.Logger) error -} - -func (i *injectable) InjectCache(c cache.Cache) error { - if i.cache == nil { - return nil - } - return i.cache(c) -} + It("should run warmup runnables before leader election is won", func(ctx SpecContext) { + By("Creating a channel to track execution order") + runnableExecutionOrderChan := make(chan string, 2) + const leaderElectionRunnableName = "leaderElectionRunnable" + const warmupRunnableName = "warmupRunnable" + + By("Creating a manager with leader election enabled") + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-warmup", + newResourceLock: fakeleaderelection.NewResourceLock, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred()) -func (i *injectable) InjectConfig(config *rest.Config) error { - if i.config == nil { - return nil - } - return i.config(config) -} + By("Creating a runnable that implements WarmupRunnable interface") + // Create a warmup runnable + warmupRunnable := newWarmupRunnableFunc( + func(ctx context.Context) error { + // This is the leader election runnable that will be executed after leader election + // It will block until context is done/cancelled + <-ctx.Done() + return nil + }, + func(ctx context.Context) error { + // This should be called during startup before leader election + runnableExecutionOrderChan <- warmupRunnableName + return nil + }, + ) + Expect(m.Add(warmupRunnable)).To(Succeed()) + + By("Creating a runnable that requires leader election") + leaderElectionRunnable := RunnableFunc( + func(ctx context.Context) error { + runnableExecutionOrderChan <- leaderElectionRunnableName + <-ctx.Done() + return nil + }, + ) + Expect(m.Add(leaderElectionRunnable)).To(Succeed()) -func (i *injectable) InjectClient(c client.Client) error { - if i.client == nil { - return nil - } - return i.client(c) -} + cm, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + resourceLockWithHooks, ok := cm.resourceLock.(fakeleaderelection.ControllableResourceLockInterface) + Expect(ok).To(BeTrue()) -func (i *injectable) InjectScheme(scheme *runtime.Scheme) error { - if i.scheme == nil { - return nil - } - return i.scheme(scheme) -} + By("Blocking leader election") + resourceLockWithHooks.BlockLeaderElection() -func (i *injectable) InjectFunc(f inject.Func) error { - if i.f == nil { - return nil - } - return i.f(f) -} + By("Starting the manager") + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).To(Succeed()) + }() -func (i *injectable) InjectStopChannel(stop <-chan struct{}) error { - if i.stop == nil { - return nil - } - return i.stop(stop) -} + By("Waiting for the warmup runnable to be executed without leader election being won") + Expect(<-runnableExecutionOrderChan).To(Equal(warmupRunnableName)) -func (i *injectable) InjectLogger(log logr.Logger) error { - if i.log == nil { - return nil - } - return i.log(log) -} + By("Unblocking leader election") + resourceLockWithHooks.UnblockLeaderElection() -func (i *injectable) Start(<-chan struct{}) error { - return nil -} + By("Waiting for the leader election runnable to be executed after leader election was won") + <-m.Elected() + Expect(<-runnableExecutionOrderChan).To(Equal(leaderElectionRunnableName)) + }) +}) type runnableError struct { } @@ -1698,18 +2027,6 @@ func (runnableError) Error() string { return "not feeling like that" } -type fakeDeferredLoader struct { - *v1alpha1.ControllerManagerConfiguration -} - -func (f *fakeDeferredLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { - return f.ControllerManagerConfiguration.ControllerManagerConfigurationSpec, nil -} - -func (f *fakeDeferredLoader) InjectScheme(scheme *runtime.Scheme) error { - return nil -} - var _ Runnable = &cacheProvider{} type cacheProvider struct { @@ -1725,36 +2042,62 @@ func (c *cacheProvider) Start(ctx context.Context) error { } type startSignalingInformer struct { + mu sync.Mutex + // The manager calls Start and WaitForCacheSync in // parallel, so we have to protect wasStarted with a Mutex // and block in WaitForCacheSync until it is true. - wasStartedLock sync.Mutex - wasStarted bool + wasStarted bool // was synced will be true once Start was called and // WaitForCacheSync returned, just like a real cache. wasSynced bool cache.Cache } -func (c *startSignalingInformer) started() bool { - c.wasStartedLock.Lock() - defer c.wasStartedLock.Unlock() - return c.wasStarted -} - func (c *startSignalingInformer) Start(ctx context.Context) error { - c.wasStartedLock.Lock() + c.mu.Lock() c.wasStarted = true - c.wasStartedLock.Unlock() + c.mu.Unlock() return c.Cache.Start(ctx) } func (c *startSignalingInformer) WaitForCacheSync(ctx context.Context) bool { defer func() { - for !c.started() { - continue - } + c.mu.Lock() c.wasSynced = true + c.mu.Unlock() }() return c.Cache.WaitForCacheSync(ctx) } + +type startClusterAfterManager struct { + informer *startSignalingInformer +} + +func (c *startClusterAfterManager) Start(ctx context.Context) error { + return c.informer.Start(ctx) +} + +func (c *startClusterAfterManager) GetCache() cache.Cache { + return c.informer +} + +// metricsDefaultServer is used to type check the default metrics server implementation +// so we can retrieve the bind addr without having to make GetBindAddr a function on the +// metricsserver.Server interface or resort to reflection. +type metricsDefaultServer interface { + GetBindAddr() string +} + +type needElection struct { + ch chan struct{} +} + +func (n *needElection) Start(_ context.Context) error { + n.ch <- struct{}{} + return nil +} + +func (n *needElection) NeedLeaderElection() bool { + return true +} diff --git a/pkg/manager/runnable_group.go b/pkg/manager/runnable_group.go new file mode 100644 index 0000000000..53e29fc56f --- /dev/null +++ b/pkg/manager/runnable_group.go @@ -0,0 +1,371 @@ +package manager + +import ( + "context" + "errors" + "sync" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + errRunnableGroupStopped = errors.New("can't accept new runnable as stop procedure is already engaged") +) + +// readyRunnable encapsulates a runnable with +// a ready check. +type readyRunnable struct { + Runnable + Check runnableCheck + signalReady bool +} + +// runnableCheck can be passed to Add() to let the runnable group determine that a +// runnable is ready. A runnable check should block until a runnable is ready, +// if the returned result is false, the runnable is considered not ready and failed. +type runnableCheck func(ctx context.Context) bool + +// runnables handles all the runnables for a manager by grouping them accordingly to their +// type (webhooks, caches etc.). +type runnables struct { + HTTPServers *runnableGroup + Webhooks *runnableGroup + Caches *runnableGroup + LeaderElection *runnableGroup + Warmup *runnableGroup + Others *runnableGroup +} + +// newRunnables creates a new runnables object. +func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables { + return &runnables{ + HTTPServers: newRunnableGroup(baseContext, errChan), + Webhooks: newRunnableGroup(baseContext, errChan), + Caches: newRunnableGroup(baseContext, errChan), + LeaderElection: newRunnableGroup(baseContext, errChan), + Warmup: newRunnableGroup(baseContext, errChan), + Others: newRunnableGroup(baseContext, errChan), + } +} + +// withLogger returns the runnables with the logger set for all runnable groups. +func (r *runnables) withLogger(logger logr.Logger) *runnables { + r.HTTPServers.withLogger(logger) + r.Webhooks.withLogger(logger) + r.Caches.withLogger(logger) + r.LeaderElection.withLogger(logger) + r.Others.withLogger(logger) + return r +} + +// Add adds a runnable to closest group of runnable that they belong to. +// +// Add should be able to be called before and after Start, but not after StopAndWait. +// Add should return an error when called during StopAndWait. +// The runnables added before Start are started when Start is called. +// The runnables added after Start are started directly. +func (r *runnables) Add(fn Runnable) error { + switch runnable := fn.(type) { + case *Server: + if runnable.NeedLeaderElection() { + return r.LeaderElection.Add(fn, nil) + } + return r.HTTPServers.Add(fn, nil) + case hasCache: + return r.Caches.Add(fn, func(ctx context.Context) bool { + return runnable.GetCache().WaitForCacheSync(ctx) + }) + case webhook.Server: + return r.Webhooks.Add(fn, nil) + case warmupRunnable, LeaderElectionRunnable: + if warmupRunnable, ok := fn.(warmupRunnable); ok { + if err := r.Warmup.Add(RunnableFunc(warmupRunnable.Warmup), nil); err != nil { + return err + } + } + + leaderElectionRunnable, ok := fn.(LeaderElectionRunnable) + if !ok { + // If the runnable is not a LeaderElectionRunnable, add it to the leader election group for backwards compatibility + return r.LeaderElection.Add(fn, nil) + } + + if !leaderElectionRunnable.NeedLeaderElection() { + return r.Others.Add(fn, nil) + } + return r.LeaderElection.Add(fn, nil) + default: + return r.LeaderElection.Add(fn, nil) + } +} + +// runnableGroup manages a group of runnables that are +// meant to be running together until StopAndWait is called. +// +// Runnables can be added to a group after the group has started +// but not after it's stopped or while shutting down. +type runnableGroup struct { + ctx context.Context + cancel context.CancelFunc + + start sync.Mutex + startOnce sync.Once + started bool + startQueue []*readyRunnable + startReadyCh chan *readyRunnable + + stop sync.RWMutex + stopOnce sync.Once + stopped bool + + // errChan is the error channel passed by the caller + // when the group is created. + // All errors are forwarded to this channel once they occur. + errChan chan error + + // ch is the internal channel where the runnables are read off from. + ch chan *readyRunnable + + // wg is an internal sync.WaitGroup that allows us to properly stop + // and wait for all the runnables to finish before returning. + wg *sync.WaitGroup + + // logger is used for logging when errors are dropped during shutdown + logger logr.Logger +} + +func newRunnableGroup(baseContext BaseContextFunc, errChan chan error) *runnableGroup { + r := &runnableGroup{ + startReadyCh: make(chan *readyRunnable), + errChan: errChan, + ch: make(chan *readyRunnable), + wg: new(sync.WaitGroup), + logger: logr.Discard(), // Default to no-op logger + } + + r.ctx, r.cancel = context.WithCancel(baseContext()) + return r +} + +// withLogger sets the logger for this runnable group. +func (r *runnableGroup) withLogger(logger logr.Logger) { + r.logger = logger +} + +// Started returns true if the group has started. +func (r *runnableGroup) Started() bool { + r.start.Lock() + defer r.start.Unlock() + return r.started +} + +// Start starts the group and waits for all +// initially registered runnables to start. +// It can only be called once, subsequent calls have no effect. +func (r *runnableGroup) Start(ctx context.Context) error { + var retErr error + + r.startOnce.Do(func() { + defer close(r.startReadyCh) + + // Start the internal reconciler. + go r.reconcile() + + // Start the group and queue up all + // the runnables that were added prior. + r.start.Lock() + r.started = true + for _, rn := range r.startQueue { + rn.signalReady = true + r.ch <- rn + } + r.start.Unlock() + + // If we don't have any queue, return. + if len(r.startQueue) == 0 { + return + } + + // Wait for all runnables to signal. + for { + select { + case <-ctx.Done(): + if err := ctx.Err(); !errors.Is(err, context.Canceled) { + retErr = err + } + case rn := <-r.startReadyCh: + for i, existing := range r.startQueue { + if existing == rn { + // Remove the item from the start queue. + r.startQueue = append(r.startQueue[:i], r.startQueue[i+1:]...) + break + } + } + // We're done waiting if the queue is empty, return. + if len(r.startQueue) == 0 { + return + } + } + } + }) + + return retErr +} + +// reconcile is our main entrypoint for every runnable added +// to this group. Its primary job is to read off the internal channel +// and schedule runnables while tracking their state. +func (r *runnableGroup) reconcile() { + for runnable := range r.ch { + // Handle stop. + // If the shutdown has been called we want to avoid + // adding new goroutines to the WaitGroup because Wait() + // panics if Add() is called after it. + { + r.stop.RLock() + if r.stopped { + // Drop any runnables if we're stopped. + r.errChan <- errRunnableGroupStopped + r.stop.RUnlock() + continue + } + + // Why is this here? + // When StopAndWait is called, if a runnable is in the process + // of being added, we could end up in a situation where + // the WaitGroup is incremented while StopAndWait has called Wait(), + // which would result in a panic. + r.wg.Add(1) + r.stop.RUnlock() + } + + // Start the runnable. + go func(rn *readyRunnable) { + go func() { + if rn.Check(r.ctx) { + if rn.signalReady { + r.startReadyCh <- rn + } + } + }() + + // If we return, the runnable ended cleanly + // or returned an error to the channel. + // + // We should always decrement the WaitGroup here. + defer r.wg.Done() + + // Start the runnable. + if err := rn.Start(r.ctx); err != nil { + // Check if we're during the shutdown process. + r.stop.RLock() + isStopped := r.stopped + r.stop.RUnlock() + + if isStopped { + // During shutdown, try to send error first (error drain goroutine might still be running) + // but drop if it would block to prevent goroutine leaks + select { + case r.errChan <- err: + // Error sent successfully (error drain goroutine is still running) + default: + // Error drain goroutine has exited, drop error to prevent goroutine leak + if !errors.Is(err, context.Canceled) { // don't log context.Canceled errors as they are expected during shutdown + r.logger.Info("error dropped during shutdown to prevent goroutine leak", "error", err) + } + } + } else { + // During normal operation, always try to send errors (may block briefly) + r.errChan <- err + } + } + }(runnable) + } +} + +// Add should be able to be called before and after Start, but not after StopAndWait. +// Add should return an error when called during StopAndWait. +func (r *runnableGroup) Add(rn Runnable, ready runnableCheck) error { + r.stop.RLock() + if r.stopped { + r.stop.RUnlock() + return errRunnableGroupStopped + } + r.stop.RUnlock() + + if ready == nil { + ready = func(_ context.Context) bool { return true } + } + + readyRunnable := &readyRunnable{ + Runnable: rn, + Check: ready, + } + + // Handle start. + // If the overall runnable group isn't started yet + // we want to buffer the runnables and let Start() + // queue them up again later. + { + r.start.Lock() + + // Check if we're already started. + if !r.started { + // Store the runnable in the internal if not. + r.startQueue = append(r.startQueue, readyRunnable) + r.start.Unlock() + return nil + } + r.start.Unlock() + } + + // Recheck if we're stopped and hold the readlock, given that the stop and start can be called + // at the same time, we can end up in a situation where the runnable is added + // after the group is stopped and the channel is closed. + r.stop.RLock() + defer r.stop.RUnlock() + if r.stopped { + return errRunnableGroupStopped + } + + // Enqueue the runnable. + r.ch <- readyRunnable + return nil +} + +// StopAndWait waits for all the runnables to finish before returning. +func (r *runnableGroup) StopAndWait(ctx context.Context) { + r.stopOnce.Do(func() { + // Close the reconciler channel once we're done. + defer func() { + r.stop.Lock() + close(r.ch) + r.stop.Unlock() + }() + + _ = r.Start(ctx) + r.stop.Lock() + // Store the stopped variable so we don't accept any new + // runnables for the time being. + r.stopped = true + r.stop.Unlock() + + // Cancel the internal channel. + r.cancel() + + done := make(chan struct{}) + go func() { + defer close(done) + // Wait for all the runnables to finish. + r.wg.Wait() + }() + + select { + case <-done: + // We're done, exit. + case <-ctx.Done(): + // Calling context has expired, exit. + } + }) +} diff --git a/pkg/manager/runnable_group_test.go b/pkg/manager/runnable_group_test.go new file mode 100644 index 0000000000..6f9b879e0e --- /dev/null +++ b/pkg/manager/runnable_group_test.go @@ -0,0 +1,386 @@ +package manager + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/cache/informertest" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var _ = Describe("runnables", func() { + errCh := make(chan error) + + It("should be able to create a new runnables object", func() { + Expect(newRunnables(defaultBaseContext, errCh)).ToNot(BeNil()) + }) + + It("should add HTTP servers to the appropriate group", func() { + server := &Server{} + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(server)).To(Succeed()) + Expect(r.HTTPServers.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should add caches to the appropriate group", func() { + cache := &cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}} + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(cache)).To(Succeed()) + Expect(r.Caches.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should add webhooks to the appropriate group", func() { + webhook := webhook.NewServer(webhook.Options{}) + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(webhook)).To(Succeed()) + Expect(r.Webhooks.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should add any runnable to the leader election group", func() { + err := errors.New("runnable func") + runnable := RunnableFunc(func(c context.Context) error { + return err + }) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(runnable)).To(Succeed()) + Expect(r.LeaderElection.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should add WarmupRunnable to the Warmup and LeaderElection group", func() { + warmupRunnable := newWarmupRunnableFunc( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return nil }, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + Expect(r.Warmup.startQueue).To(HaveLen(1)) + Expect(r.LeaderElection.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should add WarmupRunnable that doesn't needs leader election to warmup group only", func() { + warmupRunnable := newLeaderElectionAndWarmupRunnable( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return nil }, + false, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + Expect(r.Warmup.startQueue).To(HaveLen(1)) + Expect(r.LeaderElection.startQueue).To(BeEmpty()) + Expect(r.Others.startQueue).To(HaveLen(1)) + }) + + It("should add WarmupRunnable that needs leader election to Warmup and LeaderElection group, not Others", func() { + warmupRunnable := newLeaderElectionAndWarmupRunnable( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return nil }, + true, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + Expect(r.Warmup.startQueue).To(HaveLen(1)) + Expect(r.LeaderElection.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should execute the Warmup function when Warmup group is started", func(ctx SpecContext) { + var warmupExecuted atomic.Bool + + warmupRunnable := newWarmupRunnableFunc( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { + warmupExecuted.Store(true) + return nil + }, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + // Start the Warmup group + Expect(r.Warmup.Start(ctx)).To(Succeed()) + + // Verify warmup function was called + Expect(warmupExecuted.Load()).To(BeTrue()) + }) + + It("should propagate errors from Warmup function to error channel", func(ctx SpecContext) { + expectedErr := fmt.Errorf("expected warmup error") + + warmupRunnable := newWarmupRunnableFunc( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return expectedErr }, + ) + + testErrChan := make(chan error, 1) + r := newRunnables(defaultBaseContext, testErrChan) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + // Start the Warmup group in a goroutine + go func() { + Expect(r.Warmup.Start(ctx)).To(Succeed()) + }() + + // Error from Warmup should be sent to error channel + var receivedErr error + Eventually(func() error { + select { + case receivedErr = <-testErrChan: + return receivedErr + default: + return nil + } + }).Should(Equal(expectedErr)) + }) +}) + +var _ = Describe("runnableGroup", func() { + errCh := make(chan error) + + It("should be able to add new runnables before it starts", func(ctx SpecContext) { + rg := newRunnableGroup(defaultBaseContext, errCh) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + + Expect(rg.Started()).To(BeFalse()) + }) + + It("should be able to add new runnables before and after start", func(ctx SpecContext) { + rg := newRunnableGroup(defaultBaseContext, errCh) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + Expect(rg.Start(ctx)).To(Succeed()) + Expect(rg.Started()).To(BeTrue()) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + }) + + It("should be able to add new runnables before and after start concurrently", func(ctx SpecContext) { + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(50 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + + for i := 0; i < 20; i++ { + go func(i int) { + defer GinkgoRecover() + + <-time.After(time.Duration(i) * 10 * time.Millisecond) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + }(i) + } + }) + + It("should be able to close the group and wait for all runnables to finish", func(specCtx SpecContext) { + ctx, cancel := context.WithCancel(specCtx) + + exited := ptr.To(int64(0)) + rg := newRunnableGroup(defaultBaseContext, errCh) + for i := 0; i < 10; i++ { + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + defer atomic.AddInt64(exited, 1) + <-ctx.Done() + <-time.After(time.Duration(i) * 10 * time.Millisecond) + return nil + }), nil)).To(Succeed()) + } + Expect(rg.Start(ctx)).To(Succeed()) + + // Cancel the context, asking the runnables to exit. + cancel() + rg.StopAndWait(specCtx) + + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + return nil + }), nil)).ToNot(Succeed()) + + Expect(atomic.LoadInt64(exited)).To(BeNumerically("==", 10)) + }) + + It("should be able to wait for all runnables to be ready at different intervals", func(specCtx SpecContext) { + ctx, cancel := context.WithTimeout(specCtx, 1*time.Second) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(50 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + + for i := 0; i < 20; i++ { + go func(i int) { + defer GinkgoRecover() + + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), func(_ context.Context) bool { + <-time.After(time.Duration(i) * 10 * time.Millisecond) + return true + })).To(Succeed()) + }(i) + } + }) + + It("should be able to handle adding runnables while stopping", func(specCtx SpecContext) { + ctx, cancel := context.WithTimeout(specCtx, 10*time.Second) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(1 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + go func() { + defer GinkgoRecover() + <-time.After(1 * time.Millisecond) + ctx, cancel := context.WithCancel(ctx) + cancel() + rg.StopAndWait(ctx) + }() + + for i := 0; i < 200; i++ { + go func(i int) { + defer GinkgoRecover() + + <-time.After(time.Duration(i) * time.Microsecond) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), func(_ context.Context) bool { + return true + })).To(SatisfyAny( + Succeed(), + Equal(errRunnableGroupStopped), + )) + }(i) + } + }) + + It("should not turn ready if some readiness check fail", func(specCtx SpecContext) { + ctx, cancel := context.WithTimeout(specCtx, 2*time.Second) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(50 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + + for i := 0; i < 20; i++ { + go func(i int) { + defer GinkgoRecover() + + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), func(_ context.Context) bool { + <-time.After(time.Duration(i) * 10 * time.Millisecond) + return i%2 == 0 // Return false readiness all uneven indexes. + })).To(Succeed()) + }(i) + } + }) +}) + +var _ warmupRunnable = &warmupRunnableFunc{} + +func newWarmupRunnableFunc( + startFunc func(context.Context) error, + warmupFunc func(context.Context) error, +) *warmupRunnableFunc { + return &warmupRunnableFunc{ + startFunc: startFunc, + warmupFunc: warmupFunc, + } +} + +// warmupRunnableFunc is a helper struct that implements WarmupRunnable +// for testing purposes. +type warmupRunnableFunc struct { + startFunc func(context.Context) error + warmupFunc func(context.Context) error +} + +func (r *warmupRunnableFunc) Start(ctx context.Context) error { + return r.startFunc(ctx) +} + +func (r *warmupRunnableFunc) Warmup(ctx context.Context) error { + return r.warmupFunc(ctx) +} + +var _ LeaderElectionRunnable = &leaderElectionAndWarmupRunnable{} +var _ warmupRunnable = &leaderElectionAndWarmupRunnable{} + +// leaderElectionAndWarmupRunnable implements both WarmupRunnable and LeaderElectionRunnable +type leaderElectionAndWarmupRunnable struct { + *warmupRunnableFunc + needLeaderElection bool +} + +func newLeaderElectionAndWarmupRunnable( + startFunc func(context.Context) error, + warmupFunc func(context.Context) error, + needLeaderElection bool, +) *leaderElectionAndWarmupRunnable { + return &leaderElectionAndWarmupRunnable{ + warmupRunnableFunc: &warmupRunnableFunc{ + startFunc: startFunc, + warmupFunc: warmupFunc, + }, + needLeaderElection: needLeaderElection, + } +} + +func (r leaderElectionAndWarmupRunnable) NeedLeaderElection() bool { + return r.needLeaderElection +} diff --git a/pkg/manager/server.go b/pkg/manager/server.go new file mode 100644 index 0000000000..1983165da8 --- /dev/null +++ b/pkg/manager/server.go @@ -0,0 +1,109 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "context" + "errors" + "net" + "net/http" + "time" + + crlog "sigs.k8s.io/controller-runtime/pkg/log" +) + +var ( + _ Runnable = (*Server)(nil) + _ LeaderElectionRunnable = (*Server)(nil) +) + +// Server is a general purpose HTTP server Runnable for a manager. +// It is used to serve some internal handlers for health probes and profiling, +// but it can also be used to run custom servers. +type Server struct { + // Name is an optional string that describes the purpose of the server. It is used in logs to distinguish + // among multiple servers. + Name string + + // Server is the HTTP server to run. It is required. + Server *http.Server + + // Listener is an optional listener to use. If not set, the server start a listener using the server.Addr. + // Using a listener is useful when the port reservation needs to happen in advance of this runnable starting. + Listener net.Listener + + // OnlyServeWhenLeader is an optional bool that indicates that the server should only be started when the manager is the leader. + OnlyServeWhenLeader bool + + // ShutdownTimeout is an optional duration that indicates how long to wait for the server to shutdown gracefully. If not set, + // the server will wait indefinitely for all connections to close. + ShutdownTimeout *time.Duration +} + +// Start starts the server. It will block until the server is stopped or an error occurs. +func (s *Server) Start(ctx context.Context) error { + log := crlog.FromContext(ctx) + if s.Name != "" { + log = log.WithValues("name", s.Name) + } + log = log.WithValues("addr", s.addr()) + + serverShutdown := make(chan struct{}) + go func() { + <-ctx.Done() + log.Info("shutting down server") + + shutdownCtx := context.Background() + if s.ShutdownTimeout != nil { + var shutdownCancel context.CancelFunc + shutdownCtx, shutdownCancel = context.WithTimeout(shutdownCtx, *s.ShutdownTimeout) + defer shutdownCancel() + } + + if err := s.Server.Shutdown(shutdownCtx); err != nil { + log.Error(err, "error shutting down server") + } + close(serverShutdown) + }() + + log.Info("starting server") + if err := s.serve(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + + <-serverShutdown + return nil +} + +// NeedLeaderElection returns true if the server should only be started when the manager is the leader. +func (s *Server) NeedLeaderElection() bool { + return s.OnlyServeWhenLeader +} + +func (s *Server) addr() string { + if s.Listener != nil { + return s.Listener.Addr().String() + } + return s.Server.Addr +} + +func (s *Server) serve() error { + if s.Listener != nil { + return s.Server.Serve(s.Listener) + } + return s.Server.ListenAndServe() +} diff --git a/pkg/manager/signals/signal.go b/pkg/manager/signals/signal.go index 9a85558f82..a79cfb42df 100644 --- a/pkg/manager/signals/signal.go +++ b/pkg/manager/signals/signal.go @@ -24,8 +24,8 @@ import ( var onlyOneSignalHandler = make(chan struct{}) -// SetupSignalHandler registers for SIGTERM and SIGINT. A stop channel is returned -// which is closed on one of these signals. If a second signal is caught, the program +// SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned +// which is canceled on one of these signals. If a second signal is caught, the program // is terminated with exit code 1. func SetupSignalHandler() context.Context { close(onlyOneSignalHandler) // panics when called twice diff --git a/pkg/manager/signals/signal_posix.go b/pkg/manager/signals/signal_posix.go index 9bdb4e7418..a0f00a7321 100644 --- a/pkg/manager/signals/signal_posix.go +++ b/pkg/manager/signals/signal_posix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows /* diff --git a/pkg/manager/signals/signal_test.go b/pkg/manager/signals/signal_test.go index 2776e13a6d..134937e012 100644 --- a/pkg/manager/signals/signal_test.go +++ b/pkg/manager/signals/signal_test.go @@ -23,7 +23,7 @@ import ( "sync" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/pkg/manager/signals/signals_suite_test.go b/pkg/manager/signals/signals_suite_test.go index 770df0ca9c..bae6d72ed5 100644 --- a/pkg/manager/signals/signals_suite_test.go +++ b/pkg/manager/signals/signals_suite_test.go @@ -20,15 +20,13 @@ import ( "os/signal" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Runtime Signal Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Runtime Signal Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/manager/testdata/custom-config.yaml b/pkg/manager/testdata/custom-config.yaml new file mode 100644 index 0000000000..a15c9f8e5c --- /dev/null +++ b/pkg/manager/testdata/custom-config.yaml @@ -0,0 +1,3 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: CustomControllerManagerConfiguration +customValue: foo diff --git a/pkg/metrics/client_go_adapter.go b/pkg/metrics/client_go_adapter.go index d32ce25348..ff28998c44 100644 --- a/pkg/metrics/client_go_adapter.go +++ b/pkg/metrics/client_go_adapter.go @@ -18,11 +18,8 @@ package metrics import ( "context" - "net/url" - "time" "github.com/prometheus/client_golang/prometheus" - reflectormetrics "k8s.io/client-go/tools/cache" clientmetrics "k8s.io/client-go/tools/metrics" ) @@ -30,115 +27,20 @@ import ( // that client-go registers metrics. We copy the names and formats // from Kubernetes so that we match the core controllers. -// Metrics subsystem and all of the keys used by the rest client. -const ( - RestClientSubsystem = "rest_client" - LatencyKey = "request_latency_seconds" - ResultKey = "requests_total" -) - -// Metrics subsystem and all keys used by the reflectors. -const ( - ReflectorSubsystem = "reflector" - ListsTotalKey = "lists_total" - ListsDurationKey = "list_duration_seconds" - ItemsPerListKey = "items_per_list" - WatchesTotalKey = "watches_total" - ShortWatchesTotalKey = "short_watches_total" - WatchDurationKey = "watch_duration_seconds" - ItemsPerWatchKey = "items_per_watch" - LastResourceVersionKey = "last_resource_version" -) - var ( // client metrics. - // RequestLatency reports the request latency in seconds per verb/URL. - // Deprecated: This metric is deprecated for removal in a future release: using the URL as a - // dimension results in cardinality explosion for some consumers. It was deprecated upstream - // in k8s v1.14 and hidden in v1.17 via https://github.com/kubernetes/kubernetes/pull/83836. - // It is not registered by default. To register: - // import ( - // clientmetrics "k8s.io/client-go/tools/metrics" - // clmetrics "sigs.k8s.io/controller-runtime/metrics" - // ) - // - // func init() { - // clmetrics.Registry.MustRegister(clmetrics.RequestLatency) - // clientmetrics.Register(clientmetrics.RegisterOpts{ - // RequestLatency: clmetrics.LatencyAdapter - // }) - // } - RequestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: RestClientSubsystem, - Name: LatencyKey, - Help: "Request latency in seconds. Broken down by verb and URL.", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), - }, []string{"verb", "url"}) - - requestResult = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: RestClientSubsystem, - Name: ResultKey, - Help: "Number of HTTP requests, partitioned by status code, method, and host.", - }, []string{"code", "method", "host"}) - - // reflector metrics. - - // TODO(directxman12): update these to be histograms once the metrics overhaul KEP - // PRs start landing. - - listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: ReflectorSubsystem, - Name: ListsTotalKey, - Help: "Total number of API lists done by the reflectors", - }, []string{"name"}) - - listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: ListsDurationKey, - Help: "How long an API list takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: ItemsPerListKey, - Help: "How many items an API list returns to the reflectors", - }, []string{"name"}) - - watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: ReflectorSubsystem, - Name: WatchesTotalKey, - Help: "Total number of API watches done by the reflectors", - }, []string{"name"}) - - shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: ReflectorSubsystem, - Name: ShortWatchesTotalKey, - Help: "Total number of short API watches done by the reflectors", - }, []string{"name"}) - - watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: WatchDurationKey, - Help: "How long an API watch takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: ItemsPerWatchKey, - Help: "How many items an API watch returns to the reflectors", - }, []string{"name"}) - - lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: ReflectorSubsystem, - Name: LastResourceVersionKey, - Help: "Last resource version seen for the reflectors", - }, []string{"name"}) + requestResult = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rest_client_requests_total", + Help: "Number of HTTP requests, partitioned by status code, method, and host.", + }, + []string{"code", "method", "host"}, + ) ) func init() { registerClientMetrics() - registerReflectorMetrics() } // registerClientMetrics sets up the client latency metrics from client-go. @@ -152,20 +54,6 @@ func registerClientMetrics() { }) } -// registerReflectorMetrics sets up reflector (reconcile) loop metrics. -func registerReflectorMetrics() { - Registry.MustRegister(listsTotal) - Registry.MustRegister(listsDuration) - Registry.MustRegister(itemsPerList) - Registry.MustRegister(watchesTotal) - Registry.MustRegister(shortWatchesTotal) - Registry.MustRegister(watchDuration) - Registry.MustRegister(itemsPerWatch) - Registry.MustRegister(lastResourceVersion) - - reflectormetrics.SetReflectorMetricsProvider(reflectorMetricsProvider{}) -} - // this section contains adapters, implementations, and other sundry organic, artisanally // hand-crafted syntax trees required to convince client-go that it actually wants to let // someone use its metrics. @@ -174,16 +62,6 @@ func registerReflectorMetrics() { // copied (more-or-less directly) from k8s.io/kubernetes setup code // (which isn't anywhere in an easily-importable place). -// LatencyAdapter implements LatencyMetric. -type LatencyAdapter struct { - metric *prometheus.HistogramVec -} - -// Observe increments the request latency metric for the given verb/URL. -func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { - l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds()) -} - type resultAdapter struct { metric *prometheus.CounterVec } @@ -191,41 +69,3 @@ type resultAdapter struct { func (r *resultAdapter) Increment(_ context.Context, code, method, host string) { r.metric.WithLabelValues(code, method, host).Inc() } - -// Reflector metrics provider (method #2 for client-go metrics), -// copied (more-or-less directly) from k8s.io/kubernetes setup code -// (which isn't anywhere in an easily-importable place). - -type reflectorMetricsProvider struct{} - -func (reflectorMetricsProvider) NewListsMetric(name string) reflectormetrics.CounterMetric { - return listsTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewListDurationMetric(name string) reflectormetrics.SummaryMetric { - return listsDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInListMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerList.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchesMetric(name string) reflectormetrics.CounterMetric { - return watchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewShortWatchesMetric(name string) reflectormetrics.CounterMetric { - return shortWatchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchDurationMetric(name string) reflectormetrics.SummaryMetric { - return watchDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInWatchMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerWatch.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewLastResourceVersionMetric(name string) reflectormetrics.GaugeMetric { - return lastResourceVersion.WithLabelValues(name) -} diff --git a/pkg/metrics/filters/filter_suite_test.go b/pkg/metrics/filters/filter_suite_test.go new file mode 100644 index 0000000000..bdd21be491 --- /dev/null +++ b/pkg/metrics/filters/filter_suite_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "fmt" + "net/http" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +func TestSource(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Filters Suite") +} + +var testenv *envtest.Environment +var cfg *rest.Config +var clientset *kubernetes.Clientset + +// clientTransport is used to force-close keep-alives in tests that check for leaks. +var clientTransport *http.Transport + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + testenv = &envtest.Environment{} + + var err error + cfg, err = testenv.Start() + Expect(err).NotTo(HaveOccurred()) + + cfg.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { + // NB(directxman12): we can't set Transport *and* use TLS options, + // so we grab the transport right after it gets created so that we can + // type-assert on it (hopefully)? + // hopefully this doesn't break 🤞 + transport, isTransport := rt.(*http.Transport) + if !isTransport { + panic(fmt.Sprintf("wasn't able to grab underlying transport from REST client's RoundTripper, can't figure out how to close keep-alives: expected an *http.Transport, got %#v", rt)) + } + clientTransport = transport + return rt + } + + clientset, err = kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + // Prevent the metrics listener being created + metricsserver.DefaultBindAddress = "0" +}) + +var _ = AfterSuite(func() { + Expect(testenv.Stop()).To(Succeed()) + + // Put the DefaultBindAddress back + metricsserver.DefaultBindAddress = ":8080" +}) diff --git a/pkg/metrics/filters/filters.go b/pkg/metrics/filters/filters.go new file mode 100644 index 0000000000..1659502bcf --- /dev/null +++ b/pkg/metrics/filters/filters.go @@ -0,0 +1,122 @@ +package filters + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/apiserver/pkg/authentication/authenticatorfactory" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" + "k8s.io/client-go/rest" + + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// WithAuthenticationAndAuthorization provides a metrics.Filter for authentication and authorization. +// Metrics will be authenticated (via TokenReviews) and authorized (via SubjectAccessReviews) with the +// kube-apiserver. +// For the authentication and authorization the controller needs a ClusterRole +// with the following rules: +// * apiGroups: authentication.k8s.io, resources: tokenreviews, verbs: create +// * apiGroups: authorization.k8s.io, resources: subjectaccessreviews, verbs: create +// +// To scrape metrics e.g. via Prometheus the client needs a ClusterRole +// with the following rule: +// * nonResourceURLs: "/metrics", verbs: get +// +// Note: Please note that configuring this metrics provider will introduce a dependency to "k8s.io/apiserver" +// to your go module. +func WithAuthenticationAndAuthorization(config *rest.Config, httpClient *http.Client) (metricsserver.Filter, error) { + authenticationV1Client, err := authenticationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + authorizationV1Client, err := authorizationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + + authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ + Anonymous: &apiserver.AnonymousAuthConfig{Enabled: false}, // Require authentication. + CacheTTL: 1 * time.Minute, + TokenAccessReviewClient: authenticationV1Client, + TokenAccessReviewTimeout: 10 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthenticator, _, err := authenticatorConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authenticator: %w", err) + } + + authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{ + SubjectAccessReviewClient: authorizationV1Client, + AllowCacheTTL: 5 * time.Minute, + DenyCacheTTL: 30 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthorizer, err := authorizerConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authorizer: %w", err) + } + + return func(log logr.Logger, handler http.Handler) (http.Handler, error) { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + res, ok, err := delegatingAuthenticator.AuthenticateRequest(req) + if err != nil { + log.Error(err, "Authentication failed") + http.Error(w, "Authentication failed", http.StatusInternalServerError) + return + } + if !ok { + log.V(4).Info("Authentication failed") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + attributes := authorizer.AttributesRecord{ + User: res.User, + Verb: strings.ToLower(req.Method), + Path: req.URL.Path, + } + + authorized, reason, err := delegatingAuthorizer.Authorize(ctx, attributes) + if err != nil { + msg := fmt.Sprintf("Authorization for user %s failed", attributes.User.GetName()) + log.Error(err, msg) + http.Error(w, msg, http.StatusInternalServerError) + return + } + if authorized != authorizer.DecisionAllow { + msg := fmt.Sprintf("Authorization denied for user %s", attributes.User.GetName()) + log.V(4).Info(fmt.Sprintf("%s: %s", msg, reason)) + http.Error(w, msg, http.StatusForbidden) + return + } + + handler.ServeHTTP(w, req) + }), nil + }, nil +} diff --git a/pkg/metrics/filters/filters_test.go b/pkg/metrics/filters/filters_test.go new file mode 100644 index 0000000000..bd107fc56d --- /dev/null +++ b/pkg/metrics/filters/filters_test.go @@ -0,0 +1,271 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net/http" + "reflect" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +var _ = Describe("manger.Manager", func() { + Describe("Start", func() { + Context("should start serving metrics with https and authn/authz", func() { + var srv metricsserver.Server + var defaultServer metricsDefaultServer + var opts manager.Options + var httpClient *http.Client + + BeforeEach(func() { + srv = nil + newMetricsServer := func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { + var err error + srv, err = metricsserver.NewServer(options, config, httpClient) + if srv != nil { + defaultServer = srv.(metricsDefaultServer) + } + return srv, err + } + opts = manager.Options{ + Metrics: metricsserver.Options{ + BindAddress: ":0", + SecureServing: true, + FilterProvider: WithAuthenticationAndAuthorization, + }, + } + v := reflect.ValueOf(&opts).Elem() + newMetricsField := v.FieldByName("newMetricsServer") + reflect.NewAt(newMetricsField.Type(), newMetricsField.Addr().UnsafePointer()). + Elem(). + Set(reflect.ValueOf(newMetricsServer)) + httpClient = &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + }) + + It("should serve metrics in its registry", func(ctx SpecContext) { + one := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "test_one", + Help: "test metric for testing", + }) + one.Inc() + err := metrics.Registry.Register(one) + Expect(err).NotTo(HaveOccurred()) + + m, err := manager.New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) + + // Setup service account with rights to "/metrics" + token, cleanup, err := setupServiceAccountForURL(ctx, m.GetClient(), "/metrics") + defer cleanup() + Expect(err).ToNot(HaveOccurred()) + + // GET /metrics with token. + metricsEndpoint := fmt.Sprintf("https://%s/metrics", defaultServer.GetBindAddr()) + req, err := http.NewRequest("GET", metricsEndpoint, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err := httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + // This is expected as the token has rights for /metrics. + Expect(resp.StatusCode).To(Equal(200)) + + data, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring("%s\n%s\n%s\n", + `# HELP test_one test metric for testing`, + `# TYPE test_one counter`, + `test_one 1`, + )) + + // Unregister will return false if the metric was never registered + ok := metrics.Registry.Unregister(one) + Expect(ok).To(BeTrue()) + }) + + It("should serve extra endpoints", func(ctx SpecContext) { + opts.Metrics.ExtraHandlers = map[string]http.Handler{ + "/debug": http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = w.Write([]byte("Some debug info")) + }), + } + m, err := manager.New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) + + // Setup service account with rights to "/debug" + token, cleanup, err := setupServiceAccountForURL(ctx, m.GetClient(), "/debug") + defer cleanup() + Expect(err).ToNot(HaveOccurred()) + + // GET /debug without token. + endpoint := fmt.Sprintf("https://%s/debug", defaultServer.GetBindAddr()) + req, err := http.NewRequest("GET", endpoint, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err := httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + // This is expected as we didn't send a token. + Expect(resp.StatusCode).To(Equal(401)) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(body)).To(ContainSubstring("Unauthorized")) + + // PUT /debug with token. + req, err = http.NewRequest("PUT", endpoint, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err = httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + // This is expected as the token has rights for /debug. + Expect(resp.StatusCode).To(Equal(200)) + body, err = io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(body)).To(Equal("Some debug info")) + + // GET /metrics with token (but token only has rights for /debug). + metricsEndpoint := fmt.Sprintf("https://%s/metrics", defaultServer.GetBindAddr()) + req, err = http.NewRequest("GET", metricsEndpoint, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err = httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(403)) + body, err = io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + // Authorization denied is expected as the token only has rights for /debug not for /metrics. + Expect(string(body)).To(ContainSubstring("Authorization denied for user system:serviceaccount:default:metrics-test")) + }) + }) + }) +}) + +type metricsDefaultServer interface { + GetBindAddr() string +} + +func setupServiceAccountForURL(ctx context.Context, c client.Client, path string) (string, func(), error) { + createdObjects := []client.Object{} + cleanup := func() { + for _, obj := range createdObjects { + _ = c.Delete(ctx, obj) + } + } + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + Namespace: metav1.NamespaceDefault, + }, + } + if err := c.Create(ctx, sa); err != nil { + return "", cleanup, err + } + createdObjects = append(createdObjects, sa) + + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "put"}, + NonResourceURLs: []string{path}, + }, + }, + } + if err := c.Create(ctx, cr); err != nil { + return "", cleanup, err + } + createdObjects = append(createdObjects, cr) + + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "metrics-test", + Namespace: metav1.NamespaceDefault, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: "metrics-test", + }, + } + if err := c.Create(ctx, crb); err != nil { + return "", cleanup, err + } + createdObjects = append(createdObjects, crb) + + tokenRequest := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: ptr.To(int64(2 * 60 * 60)), // 2 hours. + }, + } + if err := c.SubResource("token").Create(ctx, sa, tokenRequest); err != nil { + return "", cleanup, err + } + + if tokenRequest.Status.Token == "" { + return "", cleanup, errors.New("failed to get ServiceAccount token: token should not be empty") + } + + return tokenRequest.Status.Token, cleanup, nil +} diff --git a/pkg/metrics/leaderelection.go b/pkg/metrics/leaderelection.go new file mode 100644 index 0000000000..61e1009d32 --- /dev/null +++ b/pkg/metrics/leaderelection.go @@ -0,0 +1,47 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "k8s.io/client-go/tools/leaderelection" +) + +// This file is copied and adapted from k8s.io/component-base/metrics/prometheus/clientgo/leaderelection +// which registers metrics to the k8s legacy Registry. We require very +// similar functionality, but must register metrics to a different Registry. + +var ( + leaderGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "leader_election_master_status", + Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.", + }, []string{"name"}) + + leaderSlowpathCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "leader_election_slowpath_total", + Help: "Total number of slow path exercised in renewing leader leases. 'name' is the string used to identify the lease. Please make sure to group by name.", + }, []string{"name"}) +) + +func init() { + Registry.MustRegister(leaderGauge) + leaderelection.SetProvider(leaderelectionMetricsProvider{}) +} + +type leaderelectionMetricsProvider struct{} + +func (leaderelectionMetricsProvider) NewLeaderMetric() leaderelection.LeaderMetric { + return leaderElectionPrometheusAdapter{} +} + +type leaderElectionPrometheusAdapter struct{} + +func (s leaderElectionPrometheusAdapter) On(name string) { + leaderGauge.WithLabelValues(name).Set(1.0) +} + +func (s leaderElectionPrometheusAdapter) Off(name string) { + leaderGauge.WithLabelValues(name).Set(0.0) +} + +func (leaderElectionPrometheusAdapter) SlowpathExercised(name string) { + leaderSlowpathCounter.WithLabelValues(name).Inc() +} diff --git a/pkg/metrics/listener.go b/pkg/metrics/listener.go deleted file mode 100644 index d32ae58186..0000000000 --- a/pkg/metrics/listener.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "fmt" - "net" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" -) - -var log = logf.RuntimeLog.WithName("metrics") - -// DefaultBindAddress sets the default bind address for the metrics listener -// The metrics is on by default. -var DefaultBindAddress = ":8080" - -// NewListener creates a new TCP listener bound to the given address. -func NewListener(addr string) (net.Listener, error) { - if addr == "" { - // If the metrics bind address is empty, default to ":8080" - addr = DefaultBindAddress - } - - // Add a case to disable metrics altogether - if addr == "0" { - return nil, nil - } - - log.Info("metrics server is starting to listen", "addr", addr) - ln, err := net.Listen("tcp", addr) - if err != nil { - er := fmt.Errorf("error listening on %s: %w", addr, err) - log.Error(er, "metrics server failed to listen. You may want to disable the metrics server or use another port if it is due to conflicts") - return nil, er - } - return ln, nil -} diff --git a/pkg/webhook/admission/admissiontest/doc.go b/pkg/metrics/server/doc.go similarity index 69% rename from pkg/webhook/admission/admissiontest/doc.go rename to pkg/metrics/server/doc.go index b4a7a42191..4c42f6eed7 100644 --- a/pkg/webhook/admission/admissiontest/doc.go +++ b/pkg/metrics/server/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,5 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package admissiontest contains fake webhooks for validating admission webhooks -package admissiontest +/* +Package server provides the metrics server implementation. +*/ +package server + +import ( + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" +) + +var log = logf.RuntimeLog.WithName("metrics") diff --git a/pkg/metrics/server/server.go b/pkg/metrics/server/server.go new file mode 100644 index 0000000000..939c333f7a --- /dev/null +++ b/pkg/metrics/server/server.go @@ -0,0 +1,340 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/client-go/rest" + certutil "k8s.io/client-go/util/cert" + + "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + defaultMetricsEndpoint = "/metrics" +) + +// DefaultBindAddress is the default bind address for the metrics server. +var DefaultBindAddress = ":8080" + +// Server is a server that serves metrics. +type Server interface { + // AddExtraHandler adds extra handler served on path to the http server that serves metrics. + AddExtraHandler(path string, handler http.Handler) error + + // NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates + // the metrics server doesn't need leader election. + NeedLeaderElection() bool + + // Start runs the server. + // It will install the metrics related resources depending on the server configuration. + Start(ctx context.Context) error +} + +// Options are all available options for the metrics.Server +type Options struct { + // SecureServing enables serving metrics via https. + // Per default metrics will be served via http. + SecureServing bool + + // BindAddress is the bind address for the metrics server. + // It will be defaulted to ":8080" if unspecified. + // Set this to "0" to disable the metrics server. + BindAddress string + + // ExtraHandlers contains a map of handlers (by path) which will be added to the metrics server. + // This might be useful to register diagnostic endpoints e.g. pprof. + // Note that pprof endpoints are meant to be sensitive and shouldn't be exposed publicly. + // If the simple path -> handler mapping offered here is not enough, a new http + // server/listener should be added as Runnable to the manager via the Add method. + ExtraHandlers map[string]http.Handler + + // FilterProvider provides a filter which is a func that is added around + // the metrics and the extra handlers on the metrics server. + // This can be e.g. used to enforce authentication and authorization on the handlers + // endpoint by setting this field to filters.WithAuthenticationAndAuthorization. + FilterProvider func(c *rest.Config, httpClient *http.Client) (Filter, error) + + // CertDir is the directory that contains the server key and certificate. Defaults to + // /k8s-metrics-server/serving-certs. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. + // Note: If certificate or key doesn't exist a self-signed certificate will be used. + CertDir string + + // CertName is the server certificate name. Defaults to tls.crt. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. + // Note: If certificate or key doesn't exist a self-signed certificate will be used. + CertName string + + // KeyName is the server key name. Defaults to tls.key. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. + // Note: If certificate or key doesn't exist a self-signed certificate will be used. + KeyName string + + // TLSOpts is used to allow configuring the TLS config used for the server. + // This also allows providing a certificate via GetCertificate. + TLSOpts []func(*tls.Config) + + // ListenConfig contains options for listening to an address on the metric server. + ListenConfig net.ListenConfig +} + +// Filter is a func that is added around metrics and extra handlers on the metrics server. +type Filter func(log logr.Logger, handler http.Handler) (http.Handler, error) + +// NewServer constructs a new metrics.Server from the provided options. +func NewServer(o Options, config *rest.Config, httpClient *http.Client) (Server, error) { + o.setDefaults() + + // Skip server creation if metrics are disabled. + if o.BindAddress == "0" { + return nil, nil + } + + // Validate that ExtraHandlers is not overwriting the default /metrics endpoint. + if o.ExtraHandlers != nil { + if _, ok := o.ExtraHandlers[defaultMetricsEndpoint]; ok { + return nil, fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint) + } + } + + // Create the metrics filter if a FilterProvider is set. + var metricsFilter Filter + if o.FilterProvider != nil { + var err error + metricsFilter, err = o.FilterProvider(config, httpClient) + if err != nil { + return nil, fmt.Errorf("filter provider failed to create filter for the metrics server: %w", err) + } + } + + return &defaultServer{ + metricsFilter: metricsFilter, + options: o, + }, nil +} + +// defaultServer is the default implementation used for Server. +type defaultServer struct { + options Options + + // metricsFilter is a filter which is added around + // the metrics and the extra handlers on the metrics server. + metricsFilter Filter + + // mu protects access to the bindAddr field. + mu sync.RWMutex + + // bindAddr is used to store the bindAddr after the listener has been created. + // This is used during testing to figure out the port that has been chosen randomly. + bindAddr string +} + +// setDefaults does defaulting for the Server. +func (o *Options) setDefaults() { + if o.BindAddress == "" { + o.BindAddress = DefaultBindAddress + } + + if len(o.CertDir) == 0 { + o.CertDir = filepath.Join(os.TempDir(), "k8s-metrics-server", "serving-certs") + } + + if len(o.CertName) == 0 { + o.CertName = "tls.crt" + } + + if len(o.KeyName) == 0 { + o.KeyName = "tls.key" + } +} + +// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates +// the metrics server doesn't need leader election. +func (*defaultServer) NeedLeaderElection() bool { + return false +} + +// AddExtraHandler adds extra handler served on path to the http server that serves metrics. +func (s *defaultServer) AddExtraHandler(path string, handler http.Handler) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.options.ExtraHandlers == nil { + s.options.ExtraHandlers = make(map[string]http.Handler) + } + if path == defaultMetricsEndpoint { + return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint) + } + if _, found := s.options.ExtraHandlers[path]; found { + return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path) + } + s.options.ExtraHandlers[path] = handler + return nil +} + +// Start runs the server. +// It will install the metrics related resources depend on the server configuration. +func (s *defaultServer) Start(ctx context.Context) error { + log.Info("Starting metrics server") + + listener, err := s.createListener(ctx, log) + if err != nil { + return fmt.Errorf("failed to start metrics server: failed to create listener: %w", err) + } + // Storing bindAddr here so we can retrieve it during testing via GetBindAddr. + s.mu.Lock() + s.bindAddr = listener.Addr().String() + s.mu.Unlock() + + mux := http.NewServeMux() + + handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ + ErrorHandling: promhttp.HTTPErrorOnError, + }) + if s.metricsFilter != nil { + log := log.WithValues("path", defaultMetricsEndpoint) + var err error + handler, err = s.metricsFilter(log, handler) + if err != nil { + return fmt.Errorf("failed to start metrics server: failed to add metrics filter: %w", err) + } + } + // TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics + mux.Handle(defaultMetricsEndpoint, handler) + + for path, extraHandler := range s.options.ExtraHandlers { + if s.metricsFilter != nil { + log := log.WithValues("path", path) + var err error + extraHandler, err = s.metricsFilter(log, extraHandler) + if err != nil { + return fmt.Errorf("failed to start metrics server: failed to add metrics filter to extra handler for path %s: %w", path, err) + } + } + mux.Handle(path, extraHandler) + } + + log.Info("Serving metrics server", "bindAddress", s.options.BindAddress, "secure", s.options.SecureServing) + + srv := httpserver.New(mux) + + idleConnsClosed := make(chan struct{}) + go func() { + <-ctx.Done() + log.Info("Shutting down metrics server with timeout of 1 minute") + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + // Error from closing listeners, or context timeout + log.Error(err, "error shutting down the HTTP server") + } + close(idleConnsClosed) + }() + + if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed { + return err + } + + <-idleConnsClosed + return nil +} + +func (s *defaultServer) createListener(ctx context.Context, log logr.Logger) (net.Listener, error) { + if !s.options.SecureServing { + return s.options.ListenConfig.Listen(ctx, "tcp", s.options.BindAddress) + } + + cfg := &tls.Config{ + NextProtos: []string{"h2"}, + } + // fallback TLS config ready, will now mutate if passer wants full control over it + for _, op := range s.options.TLSOpts { + op(cfg) + } + + if cfg.GetCertificate == nil { + certPath := filepath.Join(s.options.CertDir, s.options.CertName) + keyPath := filepath.Join(s.options.CertDir, s.options.KeyName) + + _, certErr := os.Stat(certPath) + certExists := !os.IsNotExist(certErr) + _, keyErr := os.Stat(keyPath) + keyExists := !os.IsNotExist(keyErr) + if certExists && keyExists { + // Create the certificate watcher and + // set the config's GetCertificate on the TLSConfig + certWatcher, err := certwatcher.New(certPath, keyPath) + if err != nil { + return nil, err + } + cfg.GetCertificate = certWatcher.GetCertificate + + go func() { + if err := certWatcher.Start(ctx); err != nil { + log.Error(err, "certificate watcher error") + } + }() + } + } + + // If cfg.GetCertificate is still nil, i.e. we didn't configure a cert watcher, fallback to a self-signed certificate. + if cfg.GetCertificate == nil { + // Note: Using self-signed certificates here should be good enough. It's just important that we + // encrypt the communication. For example kube-controller-manager also uses a self-signed certificate + // for the metrics endpoint per default. + cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures("localhost", []net.IP{{127, 0, 0, 1}}, nil, "") + if err != nil { + return nil, fmt.Errorf("failed to generate self-signed certificate for metrics server: %w", err) + } + + keyPair, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, fmt.Errorf("failed to create self-signed key pair for metrics server: %w", err) + } + cfg.Certificates = []tls.Certificate{keyPair} + } + + l, err := s.options.ListenConfig.Listen(ctx, "tcp", s.options.BindAddress) + if err != nil { + return nil, err + } + + return tls.NewListener(l, cfg), nil +} + +func (s *defaultServer) GetBindAddr() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.bindAddr +} diff --git a/pkg/metrics/workqueue.go b/pkg/metrics/workqueue.go index 8ca47235da..cd7ccc773e 100644 --- a/pkg/metrics/workqueue.go +++ b/pkg/metrics/workqueue.go @@ -16,15 +16,6 @@ limitations under the License. package metrics -import ( - "github.com/prometheus/client_golang/prometheus" - "k8s.io/client-go/util/workqueue" -) - -// This file is copied and adapted from k8s.io/kubernetes/pkg/util/workqueue/prometheus -// which registers metrics to the default prometheus Registry. We require very -// similar functionality, but must register metrics to a different Registry. - // Metrics subsystem and all keys used by the workqueue. const ( WorkQueueSubsystem = "workqueue" @@ -36,95 +27,3 @@ const ( LongestRunningProcessorKey = "longest_running_processor_seconds" RetriesKey = "retries_total" ) - -var ( - depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: DepthKey, - Help: "Current depth of workqueue", - }, []string{"name"}) - - adds = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: WorkQueueSubsystem, - Name: AddsKey, - Help: "Total number of adds handled by workqueue", - }, []string{"name"}) - - latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: WorkQueueSubsystem, - Name: QueueLatencyKey, - Help: "How long in seconds an item stays in workqueue before being requested", - Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), - }, []string{"name"}) - - workDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: WorkQueueSubsystem, - Name: WorkDurationKey, - Help: "How long in seconds processing an item from workqueue takes.", - Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), - }, []string{"name"}) - - unfinished = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: UnfinishedWorkKey, - Help: "How many seconds of work has been done that " + - "is in progress and hasn't been observed by work_duration. Large " + - "values indicate stuck threads. One can deduce the number of stuck " + - "threads by observing the rate at which this increases.", - }, []string{"name"}) - - longestRunningProcessor = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: LongestRunningProcessorKey, - Help: "How many seconds has the longest running " + - "processor for workqueue been running.", - }, []string{"name"}) - - retries = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: WorkQueueSubsystem, - Name: RetriesKey, - Help: "Total number of retries handled by workqueue", - }, []string{"name"}) -) - -func init() { - Registry.MustRegister(depth) - Registry.MustRegister(adds) - Registry.MustRegister(latency) - Registry.MustRegister(workDuration) - Registry.MustRegister(unfinished) - Registry.MustRegister(longestRunningProcessor) - Registry.MustRegister(retries) - - workqueue.SetProvider(workqueueMetricsProvider{}) -} - -type workqueueMetricsProvider struct{} - -func (workqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { - return depth.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { - return adds.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { - return latency.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { - return workDuration.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { - return unfinished.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { - return longestRunningProcessor.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { - return retries.WithLabelValues(name) -} diff --git a/pkg/patterns/application/doc.go b/pkg/patterns/application/doc.go deleted file mode 100644 index 5784051b96..0000000000 --- a/pkg/patterns/application/doc.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package application documents patterns for building Controllers to manage specific applications. -// -// -// An application is a Controller and Resource that together implement the operational logic for an application. -// They are often used to take off-the-shelf OSS applications, and make them Kubernetes native. -// -// A typical application Controller may use builder.ControllerManagedBy() to create a Controller -// for a single API type that manages other objects it creates. -// -// Application Controllers are most useful for stateful applications such as Cassandra, Etcd and MySQL -// which contain operation logic for sharding, backup and restore, upgrade / downgrade, etc. -package application diff --git a/pkg/predicate/predicate.go b/pkg/predicate/predicate.go index fc59d89ba3..9f24cb178c 100644 --- a/pkg/predicate/predicate.go +++ b/pkg/predicate/predicate.go @@ -17,6 +17,7 @@ limitations under the License. package predicate import ( + "maps" "reflect" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,44 +30,53 @@ import ( var log = logf.RuntimeLog.WithName("predicate").WithName("eventFilters") // Predicate filters events before enqueuing the keys. -type Predicate interface { +type Predicate = TypedPredicate[client.Object] + +// TypedPredicate filters events before enqueuing the keys. +type TypedPredicate[object any] interface { // Create returns true if the Create event should be processed - Create(event.CreateEvent) bool + Create(event.TypedCreateEvent[object]) bool // Delete returns true if the Delete event should be processed - Delete(event.DeleteEvent) bool + Delete(event.TypedDeleteEvent[object]) bool // Update returns true if the Update event should be processed - Update(event.UpdateEvent) bool + Update(event.TypedUpdateEvent[object]) bool // Generic returns true if the Generic event should be processed - Generic(event.GenericEvent) bool + Generic(event.TypedGenericEvent[object]) bool } -var _ Predicate = Funcs{} -var _ Predicate = ResourceVersionChangedPredicate{} -var _ Predicate = GenerationChangedPredicate{} -var _ Predicate = AnnotationChangedPredicate{} -var _ Predicate = or{} -var _ Predicate = and{} +var ( + _ Predicate = Funcs{} + _ Predicate = ResourceVersionChangedPredicate{} + _ Predicate = GenerationChangedPredicate{} + _ Predicate = AnnotationChangedPredicate{} + _ Predicate = or[client.Object]{} + _ Predicate = and[client.Object]{} + _ Predicate = not[client.Object]{} +) // Funcs is a function that implements Predicate. -type Funcs struct { +type Funcs = TypedFuncs[client.Object] + +// TypedFuncs is a function that implements TypedPredicate. +type TypedFuncs[object any] struct { // Create returns true if the Create event should be processed - CreateFunc func(event.CreateEvent) bool + CreateFunc func(event.TypedCreateEvent[object]) bool // Delete returns true if the Delete event should be processed - DeleteFunc func(event.DeleteEvent) bool + DeleteFunc func(event.TypedDeleteEvent[object]) bool // Update returns true if the Update event should be processed - UpdateFunc func(event.UpdateEvent) bool + UpdateFunc func(event.TypedUpdateEvent[object]) bool // Generic returns true if the Generic event should be processed - GenericFunc func(event.GenericEvent) bool + GenericFunc func(event.TypedGenericEvent[object]) bool } // Create implements Predicate. -func (p Funcs) Create(e event.CreateEvent) bool { +func (p TypedFuncs[object]) Create(e event.TypedCreateEvent[object]) bool { if p.CreateFunc != nil { return p.CreateFunc(e) } @@ -74,7 +84,7 @@ func (p Funcs) Create(e event.CreateEvent) bool { } // Delete implements Predicate. -func (p Funcs) Delete(e event.DeleteEvent) bool { +func (p TypedFuncs[object]) Delete(e event.TypedDeleteEvent[object]) bool { if p.DeleteFunc != nil { return p.DeleteFunc(e) } @@ -82,7 +92,7 @@ func (p Funcs) Delete(e event.DeleteEvent) bool { } // Update implements Predicate. -func (p Funcs) Update(e event.UpdateEvent) bool { +func (p TypedFuncs[object]) Update(e event.TypedUpdateEvent[object]) bool { if p.UpdateFunc != nil { return p.UpdateFunc(e) } @@ -90,7 +100,7 @@ func (p Funcs) Update(e event.UpdateEvent) bool { } // Generic implements Predicate. -func (p Funcs) Generic(e event.GenericEvent) bool { +func (p TypedFuncs[object]) Generic(e event.TypedGenericEvent[object]) bool { if p.GenericFunc != nil { return p.GenericFunc(e) } @@ -117,18 +127,41 @@ func NewPredicateFuncs(filter func(object client.Object) bool) Funcs { } } +// NewTypedPredicateFuncs returns a predicate funcs that applies the given filter function +// on CREATE, UPDATE, DELETE and GENERIC events. For UPDATE events, the filter is applied +// to the new object. +func NewTypedPredicateFuncs[object any](filter func(object object) bool) TypedFuncs[object] { + return TypedFuncs[object]{ + CreateFunc: func(e event.TypedCreateEvent[object]) bool { + return filter(e.Object) + }, + UpdateFunc: func(e event.TypedUpdateEvent[object]) bool { + return filter(e.ObjectNew) + }, + DeleteFunc: func(e event.TypedDeleteEvent[object]) bool { + return filter(e.Object) + }, + GenericFunc: func(e event.TypedGenericEvent[object]) bool { + return filter(e.Object) + }, + } +} + // ResourceVersionChangedPredicate implements a default update predicate function on resource version change. -type ResourceVersionChangedPredicate struct { - Funcs +type ResourceVersionChangedPredicate = TypedResourceVersionChangedPredicate[client.Object] + +// TypedResourceVersionChangedPredicate implements a default update predicate function on resource version change. +type TypedResourceVersionChangedPredicate[T metav1.Object] struct { + TypedFuncs[T] } // Update implements default UpdateEvent filter for validating resource version change. -func (ResourceVersionChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedResourceVersionChangedPredicate[T]) Update(e event.TypedUpdateEvent[T]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object to update", "event", e) return false } @@ -142,7 +175,27 @@ func (ResourceVersionChangedPredicate) Update(e event.UpdateEvent) bool { // The metadata.generation field of an object is incremented by the API server when writes are made to the spec field of an object. // This allows a controller to ignore update events where the spec is unchanged, and only the metadata and/or status fields are changed. // -// For CustomResource objects the Generation is only incremented when the status subresource is enabled. +// For CustomResource objects the Generation is incremented when spec is changed, or status changed and status not modeled as subresource. +// subresource status update will not increase Generation. +// +// Caveats: +// +// * The assumption that the Generation is incremented only on writing to the spec does not hold for all APIs. +// E.g For Deployment objects the Generation is also incremented on writes to the metadata.annotations field. +// For object types other than CustomResources be sure to verify which fields will trigger a Generation increment when they are written to. +// +// * With this predicate, any update events with writes only to the status field will not be reconciled. +// So in the event that the status block is overwritten or wiped by someone else the controller will not self-correct to restore the correct status. +type GenerationChangedPredicate = TypedGenerationChangedPredicate[client.Object] + +// TypedGenerationChangedPredicate implements a default update predicate function on Generation change. +// +// This predicate will skip update events that have no change in the object's metadata.generation field. +// The metadata.generation field of an object is incremented by the API server when writes are made to the spec field of an object. +// This allows a controller to ignore update events where the spec is unchanged, and only the metadata and/or status fields are changed. +// +// For CustomResource objects the Generation is incremented when spec is changed, or status changed and status not modeled as subresource. +// subresource status update will not increase Generation. // // Caveats: // @@ -152,17 +205,17 @@ func (ResourceVersionChangedPredicate) Update(e event.UpdateEvent) bool { // // * With this predicate, any update events with writes only to the status field will not be reconciled. // So in the event that the status block is overwritten or wiped by someone else the controller will not self-correct to restore the correct status. -type GenerationChangedPredicate struct { - Funcs +type TypedGenerationChangedPredicate[object metav1.Object] struct { + TypedFuncs[object] } // Update implements default UpdateEvent filter for validating generation change. -func (GenerationChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedGenerationChangedPredicate[object]) Update(e event.TypedUpdateEvent[object]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object for update", "event", e) return false } @@ -175,29 +228,32 @@ func (GenerationChangedPredicate) Update(e event.UpdateEvent) bool { // This predicate will skip update events that have no change in the object's annotation. // It is intended to be used in conjunction with the GenerationChangedPredicate, as in the following example: // -// Controller.Watch( +// Controller.Watch( // &source.Kind{Type: v1.MyCustomKind}, -// &handler.EnqueueRequestForObject{}, +// &handler.EnqueueRequestForObject{}, // predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{})) // // This is mostly useful for controllers that needs to trigger both when the resource's generation is incremented // (i.e., when the resource' .spec changes), or an annotation changes (e.g., for a staging/alpha API). -type AnnotationChangedPredicate struct { - Funcs +type AnnotationChangedPredicate = TypedAnnotationChangedPredicate[client.Object] + +// TypedAnnotationChangedPredicate implements a default update predicate function on annotation change. +type TypedAnnotationChangedPredicate[object metav1.Object] struct { + TypedFuncs[object] } // Update implements default UpdateEvent filter for validating annotation change. -func (AnnotationChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedAnnotationChangedPredicate[object]) Update(e event.TypedUpdateEvent[object]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object for update", "event", e) return false } - return !reflect.DeepEqual(e.ObjectNew.GetAnnotations(), e.ObjectOld.GetAnnotations()) + return !maps.Equal(e.ObjectNew.GetAnnotations(), e.ObjectOld.GetAnnotations()) } // LabelChangedPredicate implements a default update predicate function on label change. @@ -205,41 +261,44 @@ func (AnnotationChangedPredicate) Update(e event.UpdateEvent) bool { // This predicate will skip update events that have no change in the object's label. // It is intended to be used in conjunction with the GenerationChangedPredicate, as in the following example: // -// Controller.Watch( +// Controller.Watch( // &source.Kind{Type: v1.MyCustomKind}, -// &handler.EnqueueRequestForObject{}, +// &handler.EnqueueRequestForObject{}, // predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})) // // This will be helpful when object's labels is carrying some extra specification information beyond object's spec, // and the controller will be triggered if any valid spec change (not only in spec, but also in labels) happens. -type LabelChangedPredicate struct { - Funcs +type LabelChangedPredicate = TypedLabelChangedPredicate[client.Object] + +// TypedLabelChangedPredicate implements a default update predicate function on label change. +type TypedLabelChangedPredicate[object metav1.Object] struct { + TypedFuncs[object] } // Update implements default UpdateEvent filter for checking label change. -func (LabelChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedLabelChangedPredicate[object]) Update(e event.TypedUpdateEvent[object]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object for update", "event", e) return false } - return !reflect.DeepEqual(e.ObjectNew.GetLabels(), e.ObjectOld.GetLabels()) + return !maps.Equal(e.ObjectNew.GetLabels(), e.ObjectOld.GetLabels()) } // And returns a composite predicate that implements a logical AND of the predicates passed to it. -func And(predicates ...Predicate) Predicate { - return and{predicates} +func And[object any](predicates ...TypedPredicate[object]) TypedPredicate[object] { + return and[object]{predicates} } -type and struct { - predicates []Predicate +type and[object any] struct { + predicates []TypedPredicate[object] } -func (a and) Create(e event.CreateEvent) bool { +func (a and[object]) Create(e event.TypedCreateEvent[object]) bool { for _, p := range a.predicates { if !p.Create(e) { return false @@ -248,7 +307,7 @@ func (a and) Create(e event.CreateEvent) bool { return true } -func (a and) Update(e event.UpdateEvent) bool { +func (a and[object]) Update(e event.TypedUpdateEvent[object]) bool { for _, p := range a.predicates { if !p.Update(e) { return false @@ -257,7 +316,7 @@ func (a and) Update(e event.UpdateEvent) bool { return true } -func (a and) Delete(e event.DeleteEvent) bool { +func (a and[object]) Delete(e event.TypedDeleteEvent[object]) bool { for _, p := range a.predicates { if !p.Delete(e) { return false @@ -266,7 +325,7 @@ func (a and) Delete(e event.DeleteEvent) bool { return true } -func (a and) Generic(e event.GenericEvent) bool { +func (a and[object]) Generic(e event.TypedGenericEvent[object]) bool { for _, p := range a.predicates { if !p.Generic(e) { return false @@ -276,15 +335,15 @@ func (a and) Generic(e event.GenericEvent) bool { } // Or returns a composite predicate that implements a logical OR of the predicates passed to it. -func Or(predicates ...Predicate) Predicate { - return or{predicates} +func Or[object any](predicates ...TypedPredicate[object]) TypedPredicate[object] { + return or[object]{predicates} } -type or struct { - predicates []Predicate +type or[object any] struct { + predicates []TypedPredicate[object] } -func (o or) Create(e event.CreateEvent) bool { +func (o or[object]) Create(e event.TypedCreateEvent[object]) bool { for _, p := range o.predicates { if p.Create(e) { return true @@ -293,7 +352,7 @@ func (o or) Create(e event.CreateEvent) bool { return false } -func (o or) Update(e event.UpdateEvent) bool { +func (o or[object]) Update(e event.TypedUpdateEvent[object]) bool { for _, p := range o.predicates { if p.Update(e) { return true @@ -302,7 +361,7 @@ func (o or) Update(e event.UpdateEvent) bool { return false } -func (o or) Delete(e event.DeleteEvent) bool { +func (o or[object]) Delete(e event.TypedDeleteEvent[object]) bool { for _, p := range o.predicates { if p.Delete(e) { return true @@ -311,7 +370,7 @@ func (o or) Delete(e event.DeleteEvent) bool { return false } -func (o or) Generic(e event.GenericEvent) bool { +func (o or[object]) Generic(e event.TypedGenericEvent[object]) bool { for _, p := range o.predicates { if p.Generic(e) { return true @@ -320,6 +379,31 @@ func (o or) Generic(e event.GenericEvent) bool { return false } +// Not returns a predicate that implements a logical NOT of the predicate passed to it. +func Not[object any](predicate TypedPredicate[object]) TypedPredicate[object] { + return not[object]{predicate} +} + +type not[object any] struct { + predicate TypedPredicate[object] +} + +func (n not[object]) Create(e event.TypedCreateEvent[object]) bool { + return !n.predicate.Create(e) +} + +func (n not[object]) Update(e event.TypedUpdateEvent[object]) bool { + return !n.predicate.Update(e) +} + +func (n not[object]) Delete(e event.TypedDeleteEvent[object]) bool { + return !n.predicate.Delete(e) +} + +func (n not[object]) Generic(e event.TypedGenericEvent[object]) bool { + return !n.predicate.Generic(e) +} + // LabelSelectorPredicate constructs a Predicate from a LabelSelector. // Only objects matching the LabelSelector will be admitted. func LabelSelectorPredicate(s metav1.LabelSelector) (Predicate, error) { @@ -331,3 +415,15 @@ func LabelSelectorPredicate(s metav1.LabelSelector) (Predicate, error) { return selector.Matches(labels.Set(o.GetLabels())) }), nil } + +func isNil(arg any) bool { + if v := reflect.ValueOf(arg); !v.IsValid() || ((v.Kind() == reflect.Ptr || + v.Kind() == reflect.Interface || + v.Kind() == reflect.Slice || + v.Kind() == reflect.Map || + v.Kind() == reflect.Chan || + v.Kind() == reflect.Func) && v.IsNil()) { + return true + } + return false +} diff --git a/pkg/predicate/predicate_suite_test.go b/pkg/predicate/predicate_suite_test.go index a03d94b17d..170594ca52 100644 --- a/pkg/predicate/predicate_suite_test.go +++ b/pkg/predicate/predicate_suite_test.go @@ -19,17 +19,15 @@ package predicate_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestPredicate(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Predicate Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Predicate Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/predicate/predicate_test.go b/pkg/predicate/predicate_test.go index 005d170a66..f322b7810b 100644 --- a/pkg/predicate/predicate_test.go +++ b/pkg/predicate/predicate_test.go @@ -17,7 +17,7 @@ limitations under the License. package predicate_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -164,7 +164,7 @@ var _ = Describe("Predicate", func() { Context("Where the old object doesn't have a ResourceVersion or metadata", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -172,7 +172,7 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectNew: new, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).Should(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).Should(BeTrue()) @@ -183,7 +183,7 @@ var _ = Describe("Predicate", func() { Context("Where the new object doesn't have a ResourceVersion or metadata", func() { It("should return false", func() { - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -191,7 +191,7 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectOld: old, + ObjectOld: oldPod, } Expect(instance.Create(event.CreateEvent{})).Should(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).Should(BeTrue()) @@ -203,14 +203,14 @@ var _ = Describe("Predicate", func() { Context("Where the ResourceVersion hasn't changed", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", ResourceVersion: "v1", }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -218,8 +218,8 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).Should(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).Should(BeTrue()) @@ -231,22 +231,22 @@ var _ = Describe("Predicate", func() { Context("Where the ResourceVersion has changed", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", ResourceVersion: "v1", }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", ResourceVersion: "v2", }} passEvt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).Should(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).Should(BeTrue()) @@ -258,23 +258,23 @@ var _ = Describe("Predicate", func() { Context("Where the objects or metadata are missing", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", ResourceVersion: "v1", }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", ResourceVersion: "v1", }} - failEvt1 := event.UpdateEvent{ObjectOld: old} - failEvt2 := event.UpdateEvent{ObjectNew: new} - failEvt3 := event.UpdateEvent{ObjectOld: old, ObjectNew: new} + failEvt1 := event.UpdateEvent{ObjectOld: oldPod} + failEvt2 := event.UpdateEvent{ObjectNew: newPod} + failEvt3 := event.UpdateEvent{ObjectOld: oldPod, ObjectNew: newPod} Expect(instance.Create(event.CreateEvent{})).Should(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).Should(BeTrue()) Expect(instance.Generic(event.GenericEvent{})).Should(BeTrue()) @@ -290,7 +290,7 @@ var _ = Describe("Predicate", func() { instance := predicate.GenerationChangedPredicate{} Context("Where the old object doesn't have a Generation or metadata", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -298,7 +298,7 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectNew: new, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -309,7 +309,7 @@ var _ = Describe("Predicate", func() { Context("Where the new object doesn't have a Generation or metadata", func() { It("should return false", func() { - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -317,7 +317,7 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectOld: old, + ObjectOld: oldPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -328,14 +328,14 @@ var _ = Describe("Predicate", func() { Context("Where the Generation hasn't changed", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", Generation: 1, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -343,8 +343,8 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -355,22 +355,22 @@ var _ = Describe("Predicate", func() { Context("Where the Generation has changed", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", Generation: 1, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", Generation: 2, }} passEvt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -382,23 +382,23 @@ var _ = Describe("Predicate", func() { Context("Where the objects or metadata are missing", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", Generation: 1, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", Generation: 1, }} - failEvt1 := event.UpdateEvent{ObjectOld: old} - failEvt2 := event.UpdateEvent{ObjectNew: new} - failEvt3 := event.UpdateEvent{ObjectOld: old, ObjectNew: new} + failEvt1 := event.UpdateEvent{ObjectOld: oldPod} + failEvt2 := event.UpdateEvent{ObjectNew: newPod} + failEvt3 := event.UpdateEvent{ObjectOld: oldPod, ObjectNew: newPod} Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) Expect(instance.Generic(event.GenericEvent{})).To(BeTrue()) @@ -412,12 +412,11 @@ var _ = Describe("Predicate", func() { // AnnotationChangedPredicate has almost identical test cases as LabelChangedPredicates, // so the duplication linter should be muted on both two test suites. - // nolint:dupl Describe("When checking an AnnotationChangedPredicate", func() { instance := predicate.AnnotationChangedPredicate{} Context("Where the old object is missing", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -427,7 +426,7 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectNew: new, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -438,7 +437,7 @@ var _ = Describe("Predicate", func() { Context("Where the new object is missing", func() { It("should return false", func() { - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -448,7 +447,7 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectOld: old, + ObjectOld: oldPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -459,21 +458,21 @@ var _ = Describe("Predicate", func() { Context("Where the annotations are empty", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", }} failEvnt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -484,7 +483,7 @@ var _ = Describe("Predicate", func() { Context("Where the annotations haven't changed", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -493,7 +492,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -503,8 +502,8 @@ var _ = Describe("Predicate", func() { }} failEvnt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -515,7 +514,7 @@ var _ = Describe("Predicate", func() { Context("Where an annotation value has changed", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -524,7 +523,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -534,8 +533,8 @@ var _ = Describe("Predicate", func() { }} passEvt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -546,7 +545,7 @@ var _ = Describe("Predicate", func() { Context("Where an annotation has been added", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -555,7 +554,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -566,8 +565,8 @@ var _ = Describe("Predicate", func() { }} passEvt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -578,7 +577,7 @@ var _ = Describe("Predicate", func() { Context("Where an annotation has been removed", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -588,7 +587,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -598,8 +597,8 @@ var _ = Describe("Predicate", func() { }} passEvt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -611,12 +610,11 @@ var _ = Describe("Predicate", func() { // LabelChangedPredicates has almost identical test cases as AnnotationChangedPredicates, // so the duplication linter should be muted on both two test suites. - // nolint:dupl Describe("When checking a LabelChangedPredicate", func() { instance := predicate.LabelChangedPredicate{} Context("Where the old object is missing", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -626,7 +624,7 @@ var _ = Describe("Predicate", func() { }} evt := event.UpdateEvent{ - ObjectNew: new, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -637,7 +635,7 @@ var _ = Describe("Predicate", func() { Context("Where the new object is missing", func() { It("should return false", func() { - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -647,7 +645,7 @@ var _ = Describe("Predicate", func() { }} evt := event.UpdateEvent{ - ObjectOld: old, + ObjectOld: oldPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -658,21 +656,21 @@ var _ = Describe("Predicate", func() { Context("Where the labels are empty", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", }} evt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -683,7 +681,7 @@ var _ = Describe("Predicate", func() { Context("Where the labels haven't changed", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -692,7 +690,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -702,8 +700,8 @@ var _ = Describe("Predicate", func() { }} evt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -714,7 +712,7 @@ var _ = Describe("Predicate", func() { Context("Where a label value has changed", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -723,7 +721,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -733,8 +731,8 @@ var _ = Describe("Predicate", func() { }} evt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -745,7 +743,7 @@ var _ = Describe("Predicate", func() { Context("Where a label has been added", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -754,7 +752,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -765,8 +763,8 @@ var _ = Describe("Predicate", func() { }} evt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -777,7 +775,7 @@ var _ = Describe("Predicate", func() { Context("Where a label has been removed", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -787,7 +785,7 @@ var _ = Describe("Predicate", func() { }, }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", @@ -797,8 +795,8 @@ var _ = Describe("Predicate", func() { }} evt := event.UpdateEvent{ - ObjectOld: old, - ObjectNew: new, + ObjectOld: oldPod, + ObjectNew: newPod, } Expect(instance.Create(event.CreateEvent{})).To(BeTrue()) Expect(instance.Delete(event.DeleteEvent{})).To(BeTrue()) @@ -827,6 +825,7 @@ var _ = Describe("Predicate", func() { } passFuncs := funcs(true) failFuncs := funcs(false) + Describe("When checking an And predicate", func() { It("should return false when one of its predicates returns false", func() { a := predicate.And(passFuncs, failFuncs) @@ -859,6 +858,22 @@ var _ = Describe("Predicate", func() { Expect(o.Generic(event.GenericEvent{})).To(BeFalse()) }) }) + Describe("When checking a Not predicate", func() { + It("should return false when its predicate returns true", func() { + n := predicate.Not(passFuncs) + Expect(n.Create(event.CreateEvent{})).To(BeFalse()) + Expect(n.Update(event.UpdateEvent{})).To(BeFalse()) + Expect(n.Delete(event.DeleteEvent{})).To(BeFalse()) + Expect(n.Generic(event.GenericEvent{})).To(BeFalse()) + }) + It("should return true when its predicate returns false", func() { + n := predicate.Not(failFuncs) + Expect(n.Create(event.CreateEvent{})).To(BeTrue()) + Expect(n.Update(event.UpdateEvent{})).To(BeTrue()) + Expect(n.Delete(event.DeleteEvent{})).To(BeTrue()) + Expect(n.Generic(event.GenericEvent{})).To(BeTrue()) + }) + }) }) Describe("NewPredicateFuncs with a namespace filter function", func() { @@ -870,42 +885,42 @@ var _ = Describe("Predicate", func() { byNamespaceFuncs := predicate.NewPredicateFuncs(byNamespaceFilter("biz")) Context("Where the namespace is matching", func() { It("should return true", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", }} - passEvt1 := event.UpdateEvent{ObjectOld: old, ObjectNew: new} - Expect(byNamespaceFuncs.Create(event.CreateEvent{Object: new})).To(BeTrue()) - Expect(byNamespaceFuncs.Delete(event.DeleteEvent{Object: old})).To(BeTrue()) - Expect(byNamespaceFuncs.Generic(event.GenericEvent{Object: new})).To(BeTrue()) + passEvt1 := event.UpdateEvent{ObjectOld: oldPod, ObjectNew: newPod} + Expect(byNamespaceFuncs.Create(event.CreateEvent{Object: newPod})).To(BeTrue()) + Expect(byNamespaceFuncs.Delete(event.DeleteEvent{Object: oldPod})).To(BeTrue()) + Expect(byNamespaceFuncs.Generic(event.GenericEvent{Object: newPod})).To(BeTrue()) Expect(byNamespaceFuncs.Update(passEvt1)).To(BeTrue()) }) }) Context("Where the namespace is not matching", func() { It("should return false", func() { - new := &corev1.Pod{ + newPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "bizz", }} - old := &corev1.Pod{ + oldPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "baz", Namespace: "biz", }} - failEvt1 := event.UpdateEvent{ObjectOld: old, ObjectNew: new} - Expect(byNamespaceFuncs.Create(event.CreateEvent{Object: new})).To(BeFalse()) - Expect(byNamespaceFuncs.Delete(event.DeleteEvent{Object: new})).To(BeFalse()) - Expect(byNamespaceFuncs.Generic(event.GenericEvent{Object: new})).To(BeFalse()) + failEvt1 := event.UpdateEvent{ObjectOld: oldPod, ObjectNew: newPod} + Expect(byNamespaceFuncs.Create(event.CreateEvent{Object: newPod})).To(BeFalse()) + Expect(byNamespaceFuncs.Delete(event.DeleteEvent{Object: newPod})).To(BeFalse()) + Expect(byNamespaceFuncs.Generic(event.GenericEvent{Object: newPod})).To(BeFalse()) Expect(byNamespaceFuncs.Update(failEvt1)).To(BeFalse()) }) }) diff --git a/pkg/reconcile/reconcile.go b/pkg/reconcile/reconcile.go index b2159c531f..88303ae781 100644 --- a/pkg/reconcile/reconcile.go +++ b/pkg/reconcile/reconcile.go @@ -18,19 +18,37 @@ package reconcile import ( "context" + "errors" + "reflect" "time" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) // Result contains the result of a Reconciler invocation. type Result struct { - // Requeue tells the Controller to requeue the reconcile key. Defaults to false. + // Requeue tells the Controller to perform a ratelimited requeue + // using the workqueues ratelimiter. Defaults to false. + // + // This setting is deprecated as it causes confusion and there is + // no good reason to use it. When waiting for an external event to + // happen, either the duration until it is supposed to happen or an + // appropriate poll interval should be used, rather than an + // interval emitted by a ratelimiter whose purpose it is to control + // retry on error. + // + // Deprecated: Use `RequeueAfter` instead. Requeue bool // RequeueAfter if greater than 0, tells the Controller to requeue the reconcile key after the Duration. // Implies that Requeue is true, there is no need to set Requeue to true at the same time as RequeueAfter. RequeueAfter time.Duration + + // Priority is the priority that will be used if the item gets re-enqueued (also if an error is returned). + // If Priority is not set the original Priority of the request is preserved. + // Note: Priority is only respected if the controller is using a priorityqueue.PriorityQueue. + Priority *int } // IsZero returns true if this result is empty. @@ -61,24 +79,24 @@ Deleting Kubernetes objects) or external Events (GitHub Webhooks, polling extern Example reconcile Logic: - * Read an object and all the Pods it owns. - * Observe that the object spec specifies 5 replicas but actual cluster contains only 1 Pod replica. - * Create 4 Pods and set their OwnerReferences to the object. +* Read an object and all the Pods it owns. +* Observe that the object spec specifies 5 replicas but actual cluster contains only 1 Pod replica. +* Create 4 Pods and set their OwnerReferences to the object. reconcile may be implemented as either a type: - type reconcile struct {} + type reconciler struct {} - func (reconcile) reconcile(controller.Request) (controller.Result, error) { + func (reconciler) Reconcile(ctx context.Context, o reconcile.Request) (reconcile.Result, error) { // Implement business logic of reading and writing objects here - return controller.Result{}, nil + return reconcile.Result{}, nil } Or as a function: - controller.Func(func(o controller.Request) (controller.Result, error) { + reconcile.Func(func(ctx context.Context, o reconcile.Request) (reconcile.Result, error) { // Implement business logic of reading and writing objects here - return controller.Result{}, nil + return reconcile.Result{}, nil }) Reconciliation is level-based, meaning action isn't driven off changes in individual Events, but instead is @@ -86,17 +104,94 @@ driven by actual cluster state read from the apiserver or a local cache. For example if responding to a Pod Delete Event, the Request won't contain that a Pod was deleted, instead the reconcile function observes this when reading the cluster state and seeing the Pod as missing. */ -type Reconciler interface { - // Reconciler performs a full reconciliation for the object referred to by the Request. - // The Controller will requeue the Request to be processed again if an error is non-nil or - // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. - Reconcile(context.Context, Request) (Result, error) +type Reconciler = TypedReconciler[Request] + +// TypedReconciler implements an API for a specific Resource by Creating, Updating or Deleting Kubernetes +// objects, or by making changes to systems external to the cluster (e.g. cloudproviders, github, etc). +// +// The request type is what event handlers put into the workqueue. The workqueue then de-duplicates identical +// requests. +type TypedReconciler[request comparable] interface { + // Reconcile performs a full reconciliation for the object referred to by the Request. + // + // If the returned error is non-nil, the Result is ignored and the request will be + // requeued using exponential backoff. The only exception is if the error is a + // TerminalError in which case no requeuing happens. + // + // If the error is nil and the returned Result has a non-zero result.RequeueAfter, the request + // will be requeued after the specified duration. + // + // If the error is nil and result.RequeueAfter is zero and result.Requeue is true, the request + // will be requeued using exponential backoff. + Reconcile(context.Context, request) (Result, error) } // Func is a function that implements the reconcile interface. -type Func func(context.Context, Request) (Result, error) +type Func = TypedFunc[Request] + +// TypedFunc is a function that implements the reconcile interface. +type TypedFunc[request comparable] func(context.Context, request) (Result, error) var _ Reconciler = Func(nil) // Reconcile implements Reconciler. -func (r Func) Reconcile(ctx context.Context, o Request) (Result, error) { return r(ctx, o) } +func (r TypedFunc[request]) Reconcile(ctx context.Context, req request) (Result, error) { + return r(ctx, req) +} + +// ObjectReconciler is a specialized version of Reconciler that acts on instances of client.Object. Each reconciliation +// event gets the associated object from Kubernetes before passing it to Reconcile. An ObjectReconciler can be used in +// Builder.Complete by calling AsReconciler. See Reconciler for more details. +type ObjectReconciler[object client.Object] interface { + Reconcile(context.Context, object) (Result, error) +} + +// AsReconciler creates a Reconciler based on the given ObjectReconciler. +func AsReconciler[object client.Object](client client.Client, rec ObjectReconciler[object]) Reconciler { + return &objectReconcilerAdapter[object]{ + objReconciler: rec, + client: client, + } +} + +type objectReconcilerAdapter[object client.Object] struct { + objReconciler ObjectReconciler[object] + client client.Client +} + +// Reconcile implements Reconciler. +func (a *objectReconcilerAdapter[object]) Reconcile(ctx context.Context, req Request) (Result, error) { + o := reflect.New(reflect.TypeOf(*new(object)).Elem()).Interface().(object) + if err := a.client.Get(ctx, req.NamespacedName, o); err != nil { + return Result{}, client.IgnoreNotFound(err) + } + + return a.objReconciler.Reconcile(ctx, o) +} + +// TerminalError is an error that will not be retried but still be logged +// and recorded in metrics. +func TerminalError(wrapped error) error { + return &terminalError{err: wrapped} +} + +type terminalError struct { + err error +} + +// Unwrap returns nil if te.err is nil. +func (te *terminalError) Unwrap() error { + return te.err +} + +func (te *terminalError) Error() string { + if te.err == nil { + return "nil terminal error" + } + return "terminal error: " + te.err.Error() +} + +func (te *terminalError) Is(target error) bool { + tp := &terminalError{} + return errors.As(target, &tp) +} diff --git a/pkg/reconcile/reconcile_suite_test.go b/pkg/reconcile/reconcile_suite_test.go index 179fb10de4..9bab444ebd 100644 --- a/pkg/reconcile/reconcile_suite_test.go +++ b/pkg/reconcile/reconcile_suite_test.go @@ -19,17 +19,15 @@ package reconcile_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestReconcile(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Reconcile Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Reconcile Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/reconcile/reconcile_test.go b/pkg/reconcile/reconcile_test.go index 26924c8fa9..bb5644b87c 100644 --- a/pkg/reconcile/reconcile_test.go +++ b/pkg/reconcile/reconcile_test.go @@ -21,12 +21,25 @@ import ( "fmt" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +type mockObjectReconciler struct { + reconcileFunc func(context.Context, *corev1.ConfigMap) (reconcile.Result, error) +} + +func (r *mockObjectReconciler) Reconcile(ctx context.Context, cm *corev1.ConfigMap) (reconcile.Result, error) { + return r.reconcileFunc(ctx, cm) +} + var _ = Describe("reconcile", func() { Describe("Result", func() { It("IsZero should return true if empty", func() { @@ -50,7 +63,7 @@ var _ = Describe("reconcile", func() { }) Describe("Func", func() { - It("should call the function with the request and return a nil error.", func() { + It("should call the function with the request and return a nil error.", func(ctx SpecContext) { request := reconcile.Request{ NamespacedName: types.NamespacedName{Name: "foo", Namespace: "bar"}, } @@ -64,12 +77,12 @@ var _ = Describe("reconcile", func() { return result, nil }) - actualResult, actualErr := instance.Reconcile(context.Background(), request) + actualResult, actualErr := instance.Reconcile(ctx, request) Expect(actualResult).To(Equal(result)) Expect(actualErr).NotTo(HaveOccurred()) }) - It("should call the function with the request and return an error.", func() { + It("should call the function with the request and return an error.", func(ctx SpecContext) { request := reconcile.Request{ NamespacedName: types.NamespacedName{Name: "foo", Namespace: "bar"}, } @@ -84,9 +97,92 @@ var _ = Describe("reconcile", func() { return result, err }) - actualResult, actualErr := instance.Reconcile(context.Background(), request) + actualResult, actualErr := instance.Reconcile(ctx, request) Expect(actualResult).To(Equal(result)) Expect(actualErr).To(Equal(err)) }) + + It("should allow unwrapping inner error from terminal error", func() { + inner := apierrors.NewGone("") + terminalError := reconcile.TerminalError(inner) + + Expect(apierrors.IsGone(terminalError)).To(BeTrue()) + }) + + It("should handle nil terminal errors properly", func() { + err := reconcile.TerminalError(nil) + Expect(err.Error()).To(Equal("nil terminal error")) + }) + }) + + Describe("AsReconciler", func() { + var testenv *envtest.Environment + var testClient client.Client + + BeforeEach(func() { + testenv = &envtest.Environment{} + + cfg, err := testenv.Start() + Expect(err).NotTo(HaveOccurred()) + + testClient, err = client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Expect(testenv.Stop()).NotTo(HaveOccurred()) + }) + + Context("with an existing object", func() { + var key client.ObjectKey + + BeforeEach(func(ctx SpecContext) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + key = client.ObjectKeyFromObject(cm) + + err := testClient.Create(ctx, cm) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should Get the object and call the ObjectReconciler", func(ctx SpecContext) { + var actual *corev1.ConfigMap + reconciler := reconcile.AsReconciler(testClient, &mockObjectReconciler{ + reconcileFunc: func(ctx context.Context, cm *corev1.ConfigMap) (reconcile.Result, error) { + actual = cm + return reconcile.Result{}, nil + }, + }) + + res, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(BeZero()) + Expect(actual).NotTo(BeNil()) + Expect(actual.ObjectMeta.Name).To(Equal(key.Name)) + Expect(actual.ObjectMeta.Namespace).To(Equal(key.Namespace)) + }) + }) + + Context("with an object that doesn't exist", func() { + It("should not call the ObjectReconciler", func(ctx SpecContext) { + called := false + reconciler := reconcile.AsReconciler(testClient, &mockObjectReconciler{ + reconcileFunc: func(ctx context.Context, cm *corev1.ConfigMap) (reconcile.Result, error) { + called = true + return reconcile.Result{}, nil + }, + }) + + key := types.NamespacedName{Namespace: "default", Name: "fake-obj"} + res, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(BeZero()) + Expect(called).To(BeFalse()) + }) + }) }) }) diff --git a/pkg/recorder/example_test.go b/pkg/recorder/example_test.go index cf1beb40c8..969420d817 100644 --- a/pkg/recorder/example_test.go +++ b/pkg/recorder/example_test.go @@ -19,6 +19,7 @@ package recorder_test import ( corev1 "k8s.io/api/core/v1" + _ "github.com/onsi/ginkgo/v2" "sigs.k8s.io/controller-runtime/pkg/recorder" ) diff --git a/pkg/runtime/doc.go b/pkg/runtime/doc.go deleted file mode 100644 index 34101b3fa4..0000000000 --- a/pkg/runtime/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package runtime contains not-quite-internal mechanisms for -// controller-runtime, plus some deprecated exports of functionality -// moved elsewhere. Most users should not need to import anything in -// pkg/runtime. -package runtime diff --git a/pkg/runtime/inject/doc.go b/pkg/runtime/inject/doc.go deleted file mode 100644 index 17c60895f0..0000000000 --- a/pkg/runtime/inject/doc.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -Package inject defines interfaces and functions for propagating dependencies from a ControllerManager to -the components registered with it. Dependencies are propagated to Reconciler, Source, EventHandler and Predicate -objects which implement the Injectable interfaces. -*/ -package inject diff --git a/pkg/runtime/inject/inject.go b/pkg/runtime/inject/inject.go deleted file mode 100644 index c8c56ba817..0000000000 --- a/pkg/runtime/inject/inject.go +++ /dev/null @@ -1,164 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package inject is used by a Manager to inject types into Sources, EventHandlers, Predicates, and Reconciles. -// Deprecated: Use manager.Options fields directly. This package will be removed in v0.10. -package inject - -import ( - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Cache is used by the ControllerManager to inject Cache into Sources, EventHandlers, Predicates, and -// Reconciles. -type Cache interface { - InjectCache(cache cache.Cache) error -} - -// CacheInto will set informers on i and return the result if it implements Cache. Returns -// false if i does not implement Cache. -func CacheInto(c cache.Cache, i interface{}) (bool, error) { - if s, ok := i.(Cache); ok { - return true, s.InjectCache(c) - } - return false, nil -} - -// APIReader is used by the Manager to inject the APIReader into necessary types. -type APIReader interface { - InjectAPIReader(client.Reader) error -} - -// APIReaderInto will set APIReader on i and return the result if it implements APIReaderInto. -// Returns false if i does not implement APIReader. -func APIReaderInto(reader client.Reader, i interface{}) (bool, error) { - if s, ok := i.(APIReader); ok { - return true, s.InjectAPIReader(reader) - } - return false, nil -} - -// Config is used by the ControllerManager to inject Config into Sources, EventHandlers, Predicates, and -// Reconciles. -type Config interface { - InjectConfig(*rest.Config) error -} - -// ConfigInto will set config on i and return the result if it implements Config. Returns -// false if i does not implement Config. -func ConfigInto(config *rest.Config, i interface{}) (bool, error) { - if s, ok := i.(Config); ok { - return true, s.InjectConfig(config) - } - return false, nil -} - -// Client is used by the ControllerManager to inject client into Sources, EventHandlers, Predicates, and -// Reconciles. -type Client interface { - InjectClient(client.Client) error -} - -// ClientInto will set client on i and return the result if it implements Client. Returns -// false if i does not implement Client. -func ClientInto(client client.Client, i interface{}) (bool, error) { - if s, ok := i.(Client); ok { - return true, s.InjectClient(client) - } - return false, nil -} - -// Scheme is used by the ControllerManager to inject Scheme into Sources, EventHandlers, Predicates, and -// Reconciles. -type Scheme interface { - InjectScheme(scheme *runtime.Scheme) error -} - -// SchemeInto will set scheme and return the result on i if it implements Scheme. Returns -// false if i does not implement Scheme. -func SchemeInto(scheme *runtime.Scheme, i interface{}) (bool, error) { - if is, ok := i.(Scheme); ok { - return true, is.InjectScheme(scheme) - } - return false, nil -} - -// Stoppable is used by the ControllerManager to inject stop channel into Sources, -// EventHandlers, Predicates, and Reconciles. -type Stoppable interface { - InjectStopChannel(<-chan struct{}) error -} - -// StopChannelInto will set stop channel on i and return the result if it implements Stoppable. -// Returns false if i does not implement Stoppable. -func StopChannelInto(stop <-chan struct{}, i interface{}) (bool, error) { - if s, ok := i.(Stoppable); ok { - return true, s.InjectStopChannel(stop) - } - return false, nil -} - -// Mapper is used to inject the rest mapper to components that may need it. -type Mapper interface { - InjectMapper(meta.RESTMapper) error -} - -// MapperInto will set the rest mapper on i and return the result if it implements Mapper. -// Returns false if i does not implement Mapper. -func MapperInto(mapper meta.RESTMapper, i interface{}) (bool, error) { - if m, ok := i.(Mapper); ok { - return true, m.InjectMapper(mapper) - } - return false, nil -} - -// Func injects dependencies into i. -type Func func(i interface{}) error - -// Injector is used by the ControllerManager to inject Func into Controllers. -type Injector interface { - InjectFunc(f Func) error -} - -// InjectorInto will set f and return the result on i if it implements Injector. Returns -// false if i does not implement Injector. -func InjectorInto(f Func, i interface{}) (bool, error) { - if ii, ok := i.(Injector); ok { - return true, ii.InjectFunc(f) - } - return false, nil -} - -// Logger is used to inject Loggers into components that need them -// and don't otherwise have opinions. -type Logger interface { - InjectLogger(l logr.Logger) error -} - -// LoggerInto will set the logger on the given object if it implements inject.Logger, -// returning true if a InjectLogger was called, and false otherwise. -func LoggerInto(l logr.Logger, i interface{}) (bool, error) { - if injectable, wantsLogger := i.(Logger); wantsLogger { - return true, injectable.InjectLogger(l) - } - return false, nil -} diff --git a/pkg/runtime/inject/inject_test.go b/pkg/runtime/inject/inject_test.go deleted file mode 100644 index bffc34ec27..0000000000 --- a/pkg/runtime/inject/inject_test.go +++ /dev/null @@ -1,331 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inject - -import ( - "fmt" - "reflect" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/cache/informertest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var instance *testSource -var uninjectable *failSource -var errInjectFail = fmt.Errorf("injection fails") -var expectedFalse = false - -var _ = Describe("runtime inject", func() { - - BeforeEach(func() { - instance = &testSource{} - uninjectable = &failSource{} - }) - - It("should set informers", func() { - injectedCache := &informertest.FakeInformers{} - - By("Validating injecting the informer") - res, err := CacheInto(injectedCache, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(injectedCache).To(Equal(instance.GetCache())) - - By("Returning false if the type does not implement inject.Cache") - res, err = CacheInto(injectedCache, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(expectedFalse)) - Expect(uninjectable.GetCache()).To(BeNil()) - - By("Returning an error if informer injection fails") - res, err = CacheInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - - }) - - It("should set config", func() { - - cfg := &rest.Config{} - - By("Validating injecting config") - res, err := ConfigInto(cfg, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(cfg).To(Equal(instance.GetConfig())) - - By("Returning false if the type does not implement inject.Config") - res, err = ConfigInto(cfg, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetConfig()).To(BeNil()) - - By("Returning an error if config injection fails") - res, err = ConfigInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set client", func() { - client, err := client.NewDelegatingClient(client.NewDelegatingClientInput{Client: fake.NewClientBuilder().Build()}) - Expect(err).NotTo(HaveOccurred()) - - By("Validating injecting client") - res, err := ClientInto(client, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(client).To(Equal(instance.GetClient())) - - By("Returning false if the type does not implement inject.Client") - res, err = ClientInto(client, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetClient()).To(BeNil()) - - By("Returning an error if client injection fails") - res, err = ClientInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set scheme", func() { - - scheme := runtime.NewScheme() - - By("Validating injecting scheme") - res, err := SchemeInto(scheme, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(scheme).To(Equal(instance.GetScheme())) - - By("Returning false if the type does not implement inject.Scheme") - res, err = SchemeInto(scheme, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetScheme()).To(BeNil()) - - By("Returning an error if scheme injection fails") - res, err = SchemeInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set stop channel", func() { - - stop := make(<-chan struct{}) - - By("Validating injecting stop channel") - res, err := StopChannelInto(stop, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(stop).To(Equal(instance.GetStop())) - - By("Returning false if the type does not implement inject.Stoppable") - res, err = StopChannelInto(stop, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetStop()).To(BeNil()) - - By("Returning an error if stop channel injection fails") - res, err = StopChannelInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set api reader", func() { - apiReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{Client: fake.NewClientBuilder().Build()}) - Expect(err).NotTo(HaveOccurred()) - - By("Validating injecting client") - res, err := APIReaderInto(apiReader, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(apiReader).To(Equal(instance.GetAPIReader())) - - By("Returning false if the type does not implement inject.Client") - res, err = APIReaderInto(apiReader, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetAPIReader()).To(BeNil()) - - By("Returning an error if client injection fails") - res, err = APIReaderInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set dependencies", func() { - - f := func(interface{}) error { return nil } - - By("Validating injecting dependencies") - res, err := InjectorInto(f, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(reflect.ValueOf(f).Pointer()).To(Equal(reflect.ValueOf(instance.GetFunc()).Pointer())) - - By("Returning false if the type does not implement inject.Injector") - res, err = InjectorInto(f, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetFunc()).To(BeNil()) - - By("Returning an error if dependencies injection fails") - res, err = InjectorInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - -}) - -type testSource struct { - scheme *runtime.Scheme - cache cache.Cache - config *rest.Config - client client.Client - apiReader client.Reader - f Func - stop <-chan struct{} -} - -func (s *testSource) InjectCache(c cache.Cache) error { - if c != nil { - s.cache = c - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectConfig(config *rest.Config) error { - if config != nil { - s.config = config - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectClient(client client.Client) error { - if client != nil { - s.client = client - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectScheme(scheme *runtime.Scheme) error { - if scheme != nil { - s.scheme = scheme - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectStopChannel(stop <-chan struct{}) error { - if stop != nil { - s.stop = stop - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectAPIReader(reader client.Reader) error { - if reader != nil { - s.apiReader = reader - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectFunc(f Func) error { - if f != nil { - s.f = f - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) GetCache() cache.Cache { - return s.cache -} - -func (s *testSource) GetConfig() *rest.Config { - return s.config -} - -func (s *testSource) GetScheme() *runtime.Scheme { - return s.scheme -} - -func (s *testSource) GetClient() client.Client { - return s.client -} - -func (s *testSource) GetAPIReader() client.Reader { - return s.apiReader -} - -func (s *testSource) GetFunc() Func { - return s.f -} - -func (s *testSource) GetStop() <-chan struct{} { - return s.stop -} - -type failSource struct { - scheme *runtime.Scheme - cache cache.Cache - config *rest.Config - client client.Client - apiReader client.Reader - f Func - stop <-chan struct{} -} - -func (s *failSource) GetCache() cache.Cache { - return s.cache -} - -func (s *failSource) GetConfig() *rest.Config { - return s.config -} - -func (s *failSource) GetScheme() *runtime.Scheme { - return s.scheme -} - -func (s *failSource) GetClient() client.Client { - return s.client -} - -func (s *failSource) GetAPIReader() client.Reader { - return s.apiReader -} - -func (s *failSource) GetFunc() Func { - return s.f -} - -func (s *failSource) GetStop() <-chan struct{} { - return s.stop -} diff --git a/pkg/scheme/scheme.go b/pkg/scheme/scheme.go index 9dc93a9b21..55ebe21773 100644 --- a/pkg/scheme/scheme.go +++ b/pkg/scheme/scheme.go @@ -21,37 +21,36 @@ limitations under the License. // Each API group should define a utility function // called AddToScheme for adding its types to a Scheme: // -// // in package myapigroupv1... -// var ( -// SchemeGroupVersion = schema.GroupVersion{Group: "my.api.group", Version: "v1"} -// SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} -// AddToScheme = SchemeBuilder.AddToScheme -// ) +// // in package myapigroupv1... +// var ( +// SchemeGroupVersion = schema.GroupVersion{Group: "my.api.group", Version: "v1"} +// SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} +// AddToScheme = SchemeBuilder.AddToScheme +// ) // -// func init() { -// SchemeBuilder.Register(&MyType{}, &MyTypeList) -// } -// var ( -// scheme *runtime.Scheme = runtime.NewScheme() -// ) +// func init() { +// SchemeBuilder.Register(&MyType{}, &MyTypeList) +// } +// var ( +// scheme *runtime.Scheme = runtime.NewScheme() +// ) // // This also true of the built-in Kubernetes types. Then, in the entrypoint for // your manager, assemble the scheme containing exactly the types you need, // panicing if scheme registration failed. For instance, if our controller needs // types from the core/v1 API group (e.g. Pod), plus types from my.api.group/v1: // -// func init() { -// utilruntime.Must(myapigroupv1.AddToScheme(scheme)) -// utilruntime.Must(kubernetesscheme.AddToScheme(scheme)) -// } -// -// func main() { -// mgr := controllers.NewManager(context.Background(), controllers.GetConfigOrDie(), manager.Options{ -// Scheme: scheme, -// }) -// // ... -// } +// func init() { +// utilruntime.Must(myapigroupv1.AddToScheme(scheme)) +// utilruntime.Must(kubernetesscheme.AddToScheme(scheme)) +// } // +// func main() { +// mgr := controllers.NewManager(context.Background(), controllers.GetConfigOrDie(), manager.Options{ +// Scheme: scheme, +// }) +// // ... +// } package scheme import ( diff --git a/pkg/scheme/scheme_suite_test.go b/pkg/scheme/scheme_suite_test.go index a11e08fa5c..36ddd9decc 100644 --- a/pkg/scheme/scheme_suite_test.go +++ b/pkg/scheme/scheme_suite_test.go @@ -19,14 +19,11 @@ package scheme_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestScheme(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Scheme Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Scheme Suite") } diff --git a/pkg/scheme/scheme_test.go b/pkg/scheme/scheme_test.go index 72f083ad4b..37c6766e6f 100644 --- a/pkg/scheme/scheme_test.go +++ b/pkg/scheme/scheme_test.go @@ -19,7 +19,7 @@ package scheme_test import ( "reflect" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" appsv1 "k8s.io/api/apps/v1" diff --git a/pkg/source/example_test.go b/pkg/source/example_test.go index d306eaf583..b596ff0a0a 100644 --- a/pkg/source/example_test.go +++ b/pkg/source/example_test.go @@ -21,15 +21,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/source" ) +var mgr manager.Manager var ctrl controller.Controller // This example Watches for Pod Events (e.g. Create / Update / Delete) and enqueues a reconcile.Request // with the Name and Namespace of the Pod. func ExampleKind() { - err := ctrl.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}) + err := ctrl.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{})) if err != nil { // handle it } @@ -41,8 +43,10 @@ func ExampleChannel() { events := make(chan event.GenericEvent) err := ctrl.Watch( - &source.Channel{Source: events}, - &handler.EnqueueRequestForObject{}, + source.Channel( + events, + &handler.EnqueueRequestForObject{}, + ), ) if err != nil { // handle it diff --git a/pkg/source/source.go b/pkg/source/source.go index a63b37c443..c2c2dc4e07 100644 --- a/pkg/source/source.go +++ b/pkg/source/source.go @@ -22,219 +22,174 @@ import ( "fmt" "sync" - "k8s.io/apimachinery/pkg/api/meta" + toolscache "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" - "sigs.k8s.io/controller-runtime/pkg/source/internal" + internal "sigs.k8s.io/controller-runtime/pkg/internal/source" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/predicate" ) -var log = logf.RuntimeLog.WithName("source") +var logInformer = logf.RuntimeLog.WithName("source").WithName("Informer") -const ( - // defaultBufferSize is the default number of event notifications that can be buffered. - defaultBufferSize = 1024 -) - -// Source is a source of events (eh.g. Create, Update, Delete operations on Kubernetes Objects, Webhook callbacks, etc) +// Source is a source of events (e.g. Create, Update, Delete operations on Kubernetes Objects, Webhook callbacks, etc) // which should be processed by event.EventHandlers to enqueue reconcile.Requests. // // * Use Kind for events originating in the cluster (e.g. Pod Create, Pod Update, Deployment Update). // -// * Use Channel for events originating outside the cluster (eh.g. GitHub Webhook callback, Polling external urls). +// * Use Channel for events originating outside the cluster (e.g. GitHub Webhook callback, Polling external urls). +// +// Users may build their own Source implementations. +type Source = TypedSource[reconcile.Request] + +// TypedSource is a generic source of events (e.g. Create, Update, Delete operations on Kubernetes Objects, Webhook callbacks, etc) +// which should be processed by event.EventHandlers to enqueue a request. +// +// * Use Kind for events originating in the cluster (e.g. Pod Create, Pod Update, Deployment Update). +// +// * Use Channel for events originating outside the cluster (e.g. GitHub Webhook callback, Polling external urls). // -// Users may build their own Source implementations. If their implementations implement any of the inject package -// interfaces, the dependencies will be injected by the Controller when Watch is called. -type Source interface { - // Start is internal and should be called only by the Controller to register an EventHandler with the Informer - // to enqueue reconcile.Requests. - Start(context.Context, handler.EventHandler, workqueue.RateLimitingInterface, ...predicate.Predicate) error +// Users may build their own Source implementations. +type TypedSource[request comparable] interface { + // Start is internal and should be called only by the Controller to start the source. + // Start must be non-blocking. + Start(context.Context, workqueue.TypedRateLimitingInterface[request]) error } // SyncingSource is a source that needs syncing prior to being usable. The controller // will call its WaitForSync prior to starting workers. -type SyncingSource interface { - Source - WaitForSync(ctx context.Context) error -} - -// NewKindWithCache creates a Source without InjectCache, so that it is assured that the given cache is used -// and not overwritten. It can be used to watch objects in a different cluster by passing the cache -// from that other cluster. -func NewKindWithCache(object client.Object, cache cache.Cache) SyncingSource { - return &kindWithCache{kind: Kind{Type: object, cache: cache}} -} +type SyncingSource = TypedSyncingSource[reconcile.Request] -type kindWithCache struct { - kind Kind -} - -func (ks *kindWithCache) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { - return ks.kind.Start(ctx, handler, queue, prct...) +// TypedSyncingSource is a source that needs syncing prior to being usable. The controller +// will call its WaitForSync prior to starting workers. +type TypedSyncingSource[request comparable] interface { + TypedSource[request] + WaitForSync(ctx context.Context) error } -func (ks *kindWithCache) WaitForSync(ctx context.Context) error { - return ks.kind.WaitForSync(ctx) +// Kind creates a KindSource with the given cache provider. +func Kind[object client.Object]( + cache cache.Cache, + obj object, + handler handler.TypedEventHandler[object, reconcile.Request], + predicates ...predicate.TypedPredicate[object], +) SyncingSource { + return TypedKind(cache, obj, handler, predicates...) } -// Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create). -type Kind struct { - // Type is the type of object to watch. e.g. &v1.Pod{} - Type client.Object - - // cache used to watch APIs - cache cache.Cache - - // started may contain an error if one was encountered during startup. If its closed and does not - // contain an error, startup and syncing finished. - started chan error - startCancel func() +// TypedKind creates a KindSource with the given cache provider. +func TypedKind[object client.Object, request comparable]( + cache cache.Cache, + obj object, + handler handler.TypedEventHandler[object, request], + predicates ...predicate.TypedPredicate[object], +) TypedSyncingSource[request] { + return &internal.Kind[object, request]{ + Type: obj, + Cache: cache, + Handler: handler, + Predicates: predicates, + } } -var _ SyncingSource = &Kind{} +var _ Source = &channel[string, reconcile.Request]{} -// Start is internal and should be called only by the Controller to register an EventHandler with the Informer -// to enqueue reconcile.Requests. -func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { - // Type should have been specified by the user. - if ks.Type == nil { - return fmt.Errorf("must specify Kind.Type") - } +// ChannelOpt allows to configure a source.Channel. +type ChannelOpt[object any, request comparable] func(*channel[object, request]) - // cache should have been injected before Start was called - if ks.cache == nil { - return fmt.Errorf("must call CacheInto on Kind before calling Start") +// WithPredicates adds the configured predicates to a source.Channel. +func WithPredicates[object any, request comparable](p ...predicate.TypedPredicate[object]) ChannelOpt[object, request] { + return func(c *channel[object, request]) { + c.predicates = append(c.predicates, p...) } - - // cache.GetInformer will block until its context is cancelled if the cache was already started and it can not - // sync that informer (most commonly due to RBAC issues). - ctx, ks.startCancel = context.WithCancel(ctx) - ks.started = make(chan error) - go func() { - // Lookup the Informer from the Cache and add an EventHandler which populates the Queue - i, err := ks.cache.GetInformer(ctx, ks.Type) - if err != nil { - kindMatchErr := &meta.NoKindMatchError{} - if errors.As(err, &kindMatchErr) { - log.Error(err, "if kind is a CRD, it should be installed before calling Start", - "kind", kindMatchErr.GroupKind) - } - ks.started <- err - return - } - i.AddEventHandler(internal.EventHandler{Queue: queue, EventHandler: handler, Predicates: prct}) - if !ks.cache.WaitForCacheSync(ctx) { - // Would be great to return something more informative here - ks.started <- errors.New("cache did not sync") - } - close(ks.started) - }() - - return nil } -func (ks *Kind) String() string { - if ks.Type != nil && ks.Type.GetObjectKind() != nil { - return fmt.Sprintf("kind source: %v", ks.Type.GetObjectKind().GroupVersionKind().String()) +// WithBufferSize configures the buffer size for a source.Channel. By +// default, the buffer size is 1024. +func WithBufferSize[object any, request comparable](bufferSize int) ChannelOpt[object, request] { + return func(c *channel[object, request]) { + c.bufferSize = &bufferSize } - return "kind source: unknown GVK" } -// WaitForSync implements SyncingSource to allow controllers to wait with starting -// workers until the cache is synced. -func (ks *Kind) WaitForSync(ctx context.Context) error { - select { - case err := <-ks.started: - return err - case <-ctx.Done(): - ks.startCancel() - return errors.New("timed out waiting for cache to be synced") - } +// Channel is used to provide a source of events originating outside the cluster +// (e.g. GitHub Webhook callback). Channel requires the user to wire the external +// source (e.g. http handler) to write GenericEvents to the underlying channel. +func Channel[object any]( + source <-chan event.TypedGenericEvent[object], + handler handler.TypedEventHandler[object, reconcile.Request], + opts ...ChannelOpt[object, reconcile.Request], +) Source { + return TypedChannel[object, reconcile.Request](source, handler, opts...) } -var _ inject.Cache = &Kind{} - -// InjectCache is internal should be called only by the Controller. InjectCache is used to inject -// the Cache dependency initialized by the ControllerManager. -func (ks *Kind) InjectCache(c cache.Cache) error { - if ks.cache == nil { - ks.cache = c +// TypedChannel is used to provide a source of events originating outside the cluster +// (e.g. GitHub Webhook callback). Channel requires the user to wire the external +// source (e.g. http handler) to write GenericEvents to the underlying channel. +func TypedChannel[object any, request comparable]( + source <-chan event.TypedGenericEvent[object], + handler handler.TypedEventHandler[object, request], + opts ...ChannelOpt[object, request], +) TypedSource[request] { + c := &channel[object, request]{ + source: source, + handler: handler, + } + for _, opt := range opts { + opt(c) } - return nil -} -var _ Source = &Channel{} + return c +} -// Channel is used to provide a source of events originating outside the cluster -// (e.g. GitHub Webhook callback). Channel requires the user to wire the external -// source (eh.g. http handler) to write GenericEvents to the underlying channel. -type Channel struct { +type channel[object any, request comparable] struct { // once ensures the event distribution goroutine will be performed only once once sync.Once - // Source is the source channel to fetch GenericEvents - Source <-chan event.GenericEvent + // source is the source channel to fetch GenericEvents + source <-chan event.TypedGenericEvent[object] - // stop is to end ongoing goroutine, and close the channels - stop <-chan struct{} + handler handler.TypedEventHandler[object, request] - // dest is the destination channels of the added event handlers - dest []chan event.GenericEvent + predicates []predicate.TypedPredicate[object] - // DestBufferSize is the specified buffer size of dest channels. - // Default to 1024 if not specified. - DestBufferSize int + bufferSize *int + + // dest is the destination channels of the added event handlers + dest []chan event.TypedGenericEvent[object] // destLock is to ensure the destination channels are safely added/removed destLock sync.Mutex } -func (cs *Channel) String() string { +func (cs *channel[object, request]) String() string { return fmt.Sprintf("channel source: %p", cs) } -var _ inject.Stoppable = &Channel{} - -// InjectStopChannel is internal should be called only by the Controller. -// It is used to inject the stop channel initialized by the ControllerManager. -func (cs *Channel) InjectStopChannel(stop <-chan struct{}) error { - if cs.stop == nil { - cs.stop = stop - } - - return nil -} - // Start implements Source and should only be called by the Controller. -func (cs *Channel) Start( +func (cs *channel[object, request]) Start( ctx context.Context, - handler handler.EventHandler, - queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { + queue workqueue.TypedRateLimitingInterface[request], +) error { // Source should have been specified by the user. - if cs.Source == nil { + if cs.source == nil { return fmt.Errorf("must specify Channel.Source") } - - // stop should have been injected before Start was called - if cs.stop == nil { - return fmt.Errorf("must call InjectStop on Channel before calling Start") + if cs.handler == nil { + return errors.New("must specify Channel.Handler") } - // use default value if DestBufferSize not specified - if cs.DestBufferSize == 0 { - cs.DestBufferSize = defaultBufferSize + if cs.bufferSize == nil { + cs.bufferSize = ptr.To(1024) } - dst := make(chan event.GenericEvent, cs.DestBufferSize) + dst := make(chan event.TypedGenericEvent[object], *cs.bufferSize) cs.destLock.Lock() cs.dest = append(cs.dest, dst) @@ -248,7 +203,7 @@ func (cs *Channel) Start( go func() { for evt := range dst { shouldHandle := true - for _, p := range prct { + for _, p := range cs.predicates { if !p.Generic(evt) { shouldHandle = false break @@ -256,7 +211,11 @@ func (cs *Channel) Start( } if shouldHandle { - handler.Generic(evt, queue) + func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + cs.handler.Generic(ctx, evt, queue) + }() } } }() @@ -264,7 +223,7 @@ func (cs *Channel) Start( return nil } -func (cs *Channel) doStop() { +func (cs *channel[object, request]) doStop() { cs.destLock.Lock() defer cs.destLock.Unlock() @@ -273,7 +232,7 @@ func (cs *Channel) doStop() { } } -func (cs *Channel) distribute(evt event.GenericEvent) { +func (cs *channel[object, request]) distribute(evt event.TypedGenericEvent[object]) { cs.destLock.Lock() defer cs.destLock.Unlock() @@ -287,14 +246,14 @@ func (cs *Channel) distribute(evt event.GenericEvent) { } } -func (cs *Channel) syncLoop(ctx context.Context) { +func (cs *channel[object, request]) syncLoop(ctx context.Context) { for { select { case <-ctx.Done(): // Close destination channels cs.doStop() return - case evt, stillOpen := <-cs.Source: + case evt, stillOpen := <-cs.source: if !stillOpen { // if the source channel is closed, we're never gonna get // anything more on it, so stop & bail @@ -309,21 +268,30 @@ func (cs *Channel) syncLoop(ctx context.Context) { // Informer is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create). type Informer struct { // Informer is the controller-runtime Informer - Informer cache.Informer + Informer cache.Informer + Handler handler.EventHandler + Predicates []predicate.Predicate } var _ Source = &Informer{} // Start is internal and should be called only by the Controller to register an EventHandler with the Informer // to enqueue reconcile.Requests. -func (is *Informer) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { +func (is *Informer) Start(ctx context.Context, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) error { // Informer should have been specified by the user. if is.Informer == nil { return fmt.Errorf("must specify Informer.Informer") } + if is.Handler == nil { + return errors.New("must specify Informer.Handler") + } - is.Informer.AddEventHandler(internal.EventHandler{Queue: queue, EventHandler: handler, Predicates: prct}) + _, err := is.Informer.AddEventHandlerWithOptions(internal.NewEventHandler(ctx, queue, is.Handler, is.Predicates), toolscache.HandlerOptions{ + Logger: &logInformer, + }) + if err != nil { + return err + } return nil } @@ -334,14 +302,16 @@ func (is *Informer) String() string { var _ Source = Func(nil) // Func is a function that implements Source. -type Func func(context.Context, handler.EventHandler, workqueue.RateLimitingInterface, ...predicate.Predicate) error +type Func = TypedFunc[reconcile.Request] + +// TypedFunc is a function that implements Source. +type TypedFunc[request comparable] func(context.Context, workqueue.TypedRateLimitingInterface[request]) error // Start implements Source. -func (f Func) Start(ctx context.Context, evt handler.EventHandler, queue workqueue.RateLimitingInterface, - pr ...predicate.Predicate) error { - return f(ctx, evt, queue, pr...) +func (f TypedFunc[request]) Start(ctx context.Context, queue workqueue.TypedRateLimitingInterface[request]) error { + return f(ctx, queue) } -func (f Func) String() string { +func (f TypedFunc[request]) String() string { return fmt.Sprintf("func source: %p", f) } diff --git a/pkg/source/source_integration_test.go b/pkg/source/source_integration_test.go index f05a154d14..cc0ba530ec 100644 --- a/pkg/source/source_integration_test.go +++ b/pkg/source/source_integration_test.go @@ -17,16 +17,17 @@ limitations under the License. package source_test import ( + "context" "fmt" "time" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -37,14 +38,14 @@ import ( ) var _ = Describe("Source", func() { - var instance1, instance2 *source.Kind + var instance1, instance2 source.Source var obj client.Object - var q workqueue.RateLimitingInterface + var q workqueue.TypedRateLimitingInterface[reconcile.Request] var c1, c2 chan interface{} var ns string count := 0 - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { // Create the namespace for the test ns = fmt.Sprintf("controller-source-kindsource-%v", count) count++ @@ -53,20 +54,16 @@ var _ = Describe("Source", func() { }, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - q = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q = workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) c1 = make(chan interface{}) c2 = make(chan interface{}) }) - JustBeforeEach(func() { - instance1 = &source.Kind{Type: obj} - Expect(inject.CacheInto(icache, instance1)).To(BeTrue()) - - instance2 = &source.Kind{Type: obj} - Expect(inject.CacheInto(icache, instance2)).To(BeTrue()) - }) - - AfterEach(func() { + AfterEach(func(ctx SpecContext) { err := clientset.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) close(c1) @@ -77,7 +74,7 @@ var _ = Describe("Source", func() { Context("for a Deployment resource", func() { obj = &appsv1.Deployment{} - It("should provide Deployment Events", func() { + It("should provide Deployment Events", func(ctx SpecContext) { var created, updated, deleted *appsv1.Deployment var err error @@ -106,17 +103,17 @@ var _ = Describe("Source", func() { // Create an event handler to verify the events newHandler := func(c chan interface{}) handler.Funcs { return handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, rli workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, evt event.CreateEvent, rli workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt }, - UpdateFunc: func(evt event.UpdateEvent, rli workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, rli workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt }, - DeleteFunc: func(evt event.DeleteEvent, rli workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, rli workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt @@ -127,8 +124,10 @@ var _ = Describe("Source", func() { handler2 := newHandler(c2) // Create 2 instances - Expect(instance1.Start(ctx, handler1, q)).To(Succeed()) - Expect(instance2.Start(ctx, handler2, q)).To(Succeed()) + instance1 = source.Kind(icache, obj, handler1) + instance2 = source.Kind(icache, obj, handler2) + Expect(instance1.Start(ctx, q)).To(Succeed()) + Expect(instance2.Start(ctx, q)).To(Succeed()) By("Creating a Deployment and expecting the CreateEvent.") created, err = client.Create(ctx, deployment, metav1.CreateOptions{}) @@ -188,7 +187,7 @@ var _ = Describe("Source", func() { Expect(ok).To(BeTrue(), fmt.Sprintf("expect %T to be %T", evt, event.DeleteEvent{})) deleteEvt.Object.SetResourceVersion("") Expect(deleteEvt.Object).To(Equal(deleted)) - }, 5) + }) }) // TODO(pwittrock): Write this test @@ -240,43 +239,50 @@ var _ = Describe("Source", func() { }) Context("for a ReplicaSet resource", func() { - It("should provide a ReplicaSet CreateEvent", func() { + It("should provide a ReplicaSet CreateEvent", func(ctx SpecContext) { c := make(chan struct{}) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Informer{Informer: depInformer} - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - var err error - rs, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(rs)) - close(c) - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected GenericEvent") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := &source.Informer{ + Informer: depInformer, + Handler: handler.Funcs{ + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + var err error + rs, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(q2).To(BeIdenticalTo(q)) + Expect(evt.Object).To(Equal(rs)) + close(c) + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected GenericEvent") + }, }, - }, q) + } + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) _, err = clientset.AppsV1().ReplicaSets("default").Create(ctx, rs, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) <-c - }, 30) + }) - It("should provide a ReplicaSet UpdateEvent", func() { + It("should provide a ReplicaSet UpdateEvent", func(ctx SpecContext) { var err error rs, err = clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -284,33 +290,40 @@ var _ = Describe("Source", func() { rs2 := rs.DeepCopy() rs2.SetLabels(map[string]string{"biz": "baz"}) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Informer{Informer: depInformer} - err = instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { - }, - UpdateFunc: func(evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - var err error - rs2, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := &source.Informer{ + Informer: depInformer, + Handler: handler.Funcs{ + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + }, + UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + var err error + rs2, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) - Expect(q2).To(Equal(q)) - Expect(evt.ObjectOld).To(Equal(rs)) + Expect(q2).To(Equal(q)) + Expect(evt.ObjectOld).To(Equal(rs)) - Expect(evt.ObjectNew).To(Equal(rs2)) + Expect(evt.ObjectNew).To(Equal(rs2)) - close(c) - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected GenericEvent") + close(c) + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected GenericEvent") + }, }, - }, q) + } + err = instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) _, err = clientset.AppsV1().ReplicaSets("default").Update(ctx, rs2, metav1.UpdateOptions{}) @@ -318,27 +331,34 @@ var _ = Describe("Source", func() { <-c }) - It("should provide a ReplicaSet DeletedEvent", func() { + It("should provide a ReplicaSet DeletedEvent", func(ctx SpecContext) { c := make(chan struct{}) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Informer{Informer: depInformer} - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - }, - DeleteFunc: func(evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Expect(q2).To(Equal(q)) - Expect(evt.Object.GetName()).To(Equal(rs.Name)) - close(c) - }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected GenericEvent") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := &source.Informer{ + Informer: depInformer, + Handler: handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + }, + DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Expect(q2).To(Equal(q)) + Expect(evt.Object.GetName()).To(Equal(rs.Name)) + close(c) + }, + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected GenericEvent") + }, }, - }, q) + } + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) err = clientset.AppsV1().ReplicaSets("default").Delete(ctx, rs.Name, metav1.DeleteOptions{}) diff --git a/pkg/source/source_suite_test.go b/pkg/source/source_suite_test.go index 9fd9671cd0..774d978ca3 100644 --- a/pkg/source/source_suite_test.go +++ b/pkg/source/source_suite_test.go @@ -20,32 +20,31 @@ import ( "context" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Source Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Source Suite") } var testenv *envtest.Environment var config *rest.Config var clientset *kubernetes.Clientset var icache cache.Cache -var ctx context.Context var cancel context.CancelFunc var _ = BeforeSuite(func() { - ctx, cancel = context.WithCancel(context.Background()) + var ctx context.Context + // Has to be derived from context.Background, as it stays valid past the BeforeSuite + ctx, cancel = context.WithCancel(context.Background()) //nolint:forbidigo logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -64,9 +63,9 @@ var _ = BeforeSuite(func() { defer GinkgoRecover() Expect(icache.Start(ctx)).NotTo(HaveOccurred()) }() -}, 60) +}) var _ = AfterSuite(func() { cancel() Expect(testenv.Stop()).To(Succeed()) -}, 5) +}) diff --git a/pkg/source/source_test.go b/pkg/source/source_test.go index 966e1bb95f..ad311d73b2 100644 --- a/pkg/source/source_test.go +++ b/pkg/source/source_test.go @@ -19,14 +19,17 @@ package source_test import ( "context" "fmt" + "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/cache/informertest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" corev1 "k8s.io/api/core/v1" @@ -53,7 +56,7 @@ var _ = Describe("Source", func() { }) Context("for a Pod resource", func() { - It("should provide a Pod CreateEvent", func() { + It("should provide a Pod CreateEvent", func(ctx SpecContext) { c := make(chan struct{}) p := &corev1.Pod{ Spec: corev1.PodSpec{ @@ -63,57 +66,58 @@ var _ = Describe("Source", func() { }, } - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(inject.CacheInto(ic, instance)).To(BeTrue()) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(ctx context.Context, evt event.TypedCreateEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt.Object).To(Equal(p)) close(c) }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.TypedUpdateEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.TypedDeleteEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.TypedGenericEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, - }, q) + }) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).NotTo(HaveOccurred()) + Expect(instance.WaitForSync(ctx)).NotTo(HaveOccurred()) - i, err := ic.FakeInformerFor(&corev1.Pod{}) + i, err := ic.FakeInformerFor(ctx, &corev1.Pod{}) Expect(err).NotTo(HaveOccurred()) i.Add(p) <-c }) - It("should provide a Pod UpdateEvent", func() { + It("should provide a Pod UpdateEvent", func(ctx SpecContext) { p2 := p.DeepCopy() p2.SetLabels(map[string]string{"biz": "baz"}) ic := &informertest.FakeInformers{} - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(instance.InjectCache(ic)).To(Succeed()) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(ctx context.Context, evt event.TypedCreateEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, evt event.TypedUpdateEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.ObjectOld).To(Equal(p)) @@ -122,26 +126,27 @@ var _ = Describe("Source", func() { close(c) }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.TypedDeleteEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.TypedGenericEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, - }, q) + }) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).NotTo(HaveOccurred()) + Expect(instance.WaitForSync(ctx)).NotTo(HaveOccurred()) - i, err := ic.FakeInformerFor(&corev1.Pod{}) + i, err := ic.FakeInformerFor(ctx, &corev1.Pod{}) Expect(err).NotTo(HaveOccurred()) i.Update(p, p2) <-c }) - It("should provide a Pod DeletedEvent", func() { + It("should provide a Pod DeletedEvent", func(ctx SpecContext) { c := make(chan struct{}) p := &corev1.Pod{ Spec: corev1.PodSpec{ @@ -151,35 +156,36 @@ var _ = Describe("Source", func() { }, } - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(inject.CacheInto(ic, instance)).To(BeTrue()) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(context.Context, event.TypedCreateEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.TypedUpdateEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, evt event.TypedDeleteEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.Object).To(Equal(p)) close(c) }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.TypedGenericEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, - }, q) + }) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).NotTo(HaveOccurred()) + Expect(instance.WaitForSync(ctx)).NotTo(HaveOccurred()) - i, err := ic.FakeInformerFor(&corev1.Pod{}) + i, err := ic.FakeInformerFor(ctx, &corev1.Pod{}) Expect(err).NotTo(HaveOccurred()) i.Delete(p) @@ -187,61 +193,60 @@ var _ = Describe("Source", func() { }) }) - It("should return an error from Start if informers were not injected", func() { - instance := source.Kind{Type: &corev1.Pod{}} - err := instance.Start(ctx, nil, nil) + It("should return an error from Start cache was not provided", func(ctx SpecContext) { + instance := source.Kind(nil, &corev1.Pod{}, nil) + err := instance.Start(ctx, nil) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("must call CacheInto on Kind before calling Start")) + Expect(err.Error()).To(ContainSubstring("must create Kind with a non-nil cache")) }) - It("should return an error from Start if a type was not provided", func() { - instance := source.Kind{} - Expect(instance.InjectCache(&informertest.FakeInformers{})).To(Succeed()) - err := instance.Start(ctx, nil, nil) + It("should return an error from Start if a type was not provided", func(ctx SpecContext) { + instance := source.Kind[client.Object](ic, nil, nil) + err := instance.Start(ctx, nil) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("must specify Kind.Type")) + Expect(err.Error()).To(ContainSubstring("must create Kind with a non-nil object")) + }) + It("should return an error from Start if a handler was not provided", func(ctx SpecContext) { + instance := source.Kind(ic, &corev1.Pod{}, nil) + err := instance.Start(ctx, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must create Kind with non-nil handler")) }) - It("should return an error if syncing fails", func() { - instance := source.Kind{Type: &corev1.Pod{}} + It("should return an error if syncing fails", func(ctx SpecContext) { f := false - Expect(instance.InjectCache(&informertest.FakeInformers{Synced: &f})).To(Succeed()) - Expect(instance.Start(context.Background(), nil, nil)).NotTo(HaveOccurred()) - err := instance.WaitForSync(context.Background()) + instance := source.Kind[client.Object](&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}, &handler.EnqueueRequestForObject{}) + Expect(instance.Start(ctx, nil)).NotTo(HaveOccurred()) + err := instance.WaitForSync(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("cache did not sync")) }) Context("for a Kind not in the cache", func() { - It("should return an error when WaitForSync is called", func() { + It("should return an error when WaitForSync is called", func(specContext SpecContext) { ic.Error = fmt.Errorf("test error") - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(instance.InjectCache(ic)).To(Succeed()) - err := instance.Start(ctx, handler.Funcs{}, q) + ctx, cancel := context.WithTimeout(specContext, 2*time.Second) + defer cancel() + + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{}) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).To(HaveOccurred()) + Eventually(instance.WaitForSync).WithArguments(ctx).Should(HaveOccurred()) }) }) - }) - - Describe("KindWithCache", func() { - It("should not allow injecting a cache", func() { - instance := source.NewKindWithCache(nil, nil) - injected, err := inject.CacheInto(&informertest.FakeInformers{}, instance) - Expect(err).To(BeNil()) - Expect(injected).To(BeFalse()) - }) - It("should return an error if syncing fails", func() { + It("should return an error if syncing fails", func(ctx SpecContext) { f := false - instance := source.NewKindWithCache(&corev1.Pod{}, &informertest.FakeInformers{Synced: &f}) - Expect(instance.Start(context.Background(), nil, nil)).NotTo(HaveOccurred()) - err := instance.WaitForSync(context.Background()) + instance := source.Kind[client.Object](&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}, &handler.EnqueueRequestForObject{}) + Expect(instance.Start(ctx, nil)).NotTo(HaveOccurred()) + err := instance.WaitForSync(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("cache did not sync")) @@ -249,46 +254,40 @@ var _ = Describe("Source", func() { }) Describe("Func", func() { - It("should be called from Start", func() { + It("should be called from Start", func(ctx SpecContext) { run := false instance := source.Func(func( context.Context, - handler.EventHandler, - workqueue.RateLimitingInterface, ...predicate.Predicate) error { + workqueue.TypedRateLimitingInterface[reconcile.Request]) error { run = true return nil }) - Expect(instance.Start(ctx, nil, nil)).NotTo(HaveOccurred()) + Expect(instance.Start(ctx, nil)).NotTo(HaveOccurred()) Expect(run).To(BeTrue()) expected := fmt.Errorf("expected error: Func") instance = source.Func(func( context.Context, - handler.EventHandler, - workqueue.RateLimitingInterface, ...predicate.Predicate) error { + workqueue.TypedRateLimitingInterface[reconcile.Request]) error { return expected }) - Expect(instance.Start(ctx, nil, nil)).To(Equal(expected)) + Expect(instance.Start(ctx, nil)).To(Equal(expected)) }) }) Describe("Channel", func() { - var ctx context.Context - var cancel context.CancelFunc var ch chan event.GenericEvent BeforeEach(func() { - ctx, cancel = context.WithCancel(context.Background()) ch = make(chan event.GenericEvent) }) AfterEach(func() { - cancel() close(ch) }) Context("for a source", func() { - It("should provide a GenericEvent", func() { + It("should provide a GenericEvent", func(ctx SpecContext) { ch := make(chan event.GenericEvent) c := make(chan struct{}) p := &corev1.Pod{ @@ -307,75 +306,87 @@ var _ = Describe("Source", func() { }, } - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{Source: ch} - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - // The empty event should have been filtered out by the predicates, - // and will not be passed to the handler. - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(p)) - close(c) + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + // The empty event should have been filtered out by the predicates, + // and will not be passed to the handler. + Expect(q2).To(BeIdenticalTo(q)) + Expect(evt.Object).To(Equal(p)) + close(c) + }, }, - }, q, prct) + source.WithPredicates[client.Object, reconcile.Request](prct), + ) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) ch <- invalidEvt ch <- evt <-c }) - It("should get pending events processed once channel unblocked", func() { + It("should get pending events processed once channel unblocked", func(ctx SpecContext) { ch := make(chan event.GenericEvent) unblock := make(chan struct{}) processed := make(chan struct{}) evt := event.GenericEvent{} eventCount := 0 - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) // Add a handler to get distribution blocked - instance := &source.Channel{Source: ch} - instance.DestBufferSize = 1 - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - // Block for the first time - if eventCount == 0 { - <-unblock - } - eventCount++ - - if eventCount == 3 { - close(processed) - } + instance := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + // Block for the first time + if eventCount == 0 { + <-unblock + } + eventCount++ + + if eventCount == 3 { + close(processed) + } + }, }, - }, q) + ) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) // Write 3 events into the source channel. @@ -396,43 +407,52 @@ var _ = Describe("Source", func() { // Validate all of the events have been processed. Expect(eventCount).To(Equal(3)) }) - It("should be able to cope with events in the channel before the source is started", func() { + It("should be able to cope with events in the channel before the source is started", func(ctx SpecContext) { ch := make(chan event.GenericEvent, 1) processed := make(chan struct{}) evt := event.GenericEvent{} ch <- evt - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) // Add a handler to get distribution blocked - instance := &source.Channel{Source: ch} - instance.DestBufferSize = 1 - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) + instance := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") + close(processed) + }, }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() + ) - close(processed) - }, - }, q) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) <-processed }) - It("should stop when the source channel is closed", func() { - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + It("should stop when the source channel is closed", func(ctx SpecContext) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) // if we didn't stop, we'd start spamming the queue with empty // messages as we "received" a zero-valued GenericEvent from // the source channel @@ -444,121 +464,48 @@ var _ = Describe("Source", func() { close(ch) By("feeding that channel to a channel source") - src := &source.Channel{Source: ch} - Expect(inject.StopChannelInto(ctx.Done(), src)).To(BeTrue()) - processed := make(chan struct{}) defer close(processed) + src := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() - err := src.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") + processed <- struct{}{} + }, }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() + ) - processed <- struct{}{} - }, - }, q) + err := src.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) By("expecting to only get one event") Eventually(processed).Should(Receive()) Consistently(processed).ShouldNot(Receive()) }) - It("should get error if no source specified", func() { - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{ /*no source specified*/ } - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) - err := instance.Start(ctx, handler.Funcs{}, q) + It("should get error if no source specified", func(ctx SpecContext) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Channel[string](nil, nil /*no source specified*/) + err := instance.Start(ctx, q) Expect(err).To(Equal(fmt.Errorf("must specify Channel.Source"))) }) - It("should get error if no stop channel injected", func() { - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{Source: ch} - err := instance.Start(ctx, handler.Funcs{}, q) - Expect(err).To(Equal(fmt.Errorf("must call InjectStop on Channel before calling Start"))) - }) - }) - Context("for multi sources (handlers)", func() { - It("should provide GenericEvents for all handlers", func() { - ch := make(chan event.GenericEvent) - p := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, - } - evt := event.GenericEvent{ - Object: p, - } - - var resEvent1, resEvent2 event.GenericEvent - c1 := make(chan struct{}) - c2 := make(chan struct{}) - - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{Source: ch} - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(p)) - resEvent1 = evt - close(c1) - }, - }, q) - Expect(err).NotTo(HaveOccurred()) - - err = instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(p)) - resEvent2 = evt - close(c2) - }, - }, q) - Expect(err).NotTo(HaveOccurred()) - - ch <- evt - <-c1 - <-c2 - - // Validate the two handlers received same event - Expect(resEvent1).To(Equal(resEvent2)) - }) }) }) }) diff --git a/pkg/webhook/admission/admission_suite_test.go b/pkg/webhook/admission/admission_suite_test.go index 3648aa45b7..f4e561b1b8 100644 --- a/pkg/webhook/admission/admission_suite_test.go +++ b/pkg/webhook/admission/admission_suite_test.go @@ -19,20 +19,18 @@ package admission import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestAdmissionWebhook(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Admission Webhook Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Admission Webhook Suite") } var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) -}, 60) +}) diff --git a/pkg/webhook/admission/admissiontest/util.go b/pkg/webhook/admission/admissiontest/util.go deleted file mode 100644 index 685e8274d8..0000000000 --- a/pkg/webhook/admission/admissiontest/util.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admissiontest - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -// FakeValidator provides fake validating webhook functionality for testing -// It implements the admission.Validator interface and -// rejects all requests with the same configured error -// or passes if ErrorToReturn is nil. -type FakeValidator struct { - // ErrorToReturn is the error for which the FakeValidator rejects all requests - ErrorToReturn error `json:"ErrorToReturn,omitempty"` - // GVKToReturn is the GroupVersionKind that the webhook operates on - GVKToReturn schema.GroupVersionKind -} - -// ValidateCreate implements admission.Validator. -func (v *FakeValidator) ValidateCreate() error { - return v.ErrorToReturn -} - -// ValidateUpdate implements admission.Validator. -func (v *FakeValidator) ValidateUpdate(old runtime.Object) error { - return v.ErrorToReturn -} - -// ValidateDelete implements admission.Validator. -func (v *FakeValidator) ValidateDelete() error { - return v.ErrorToReturn -} - -// GetObjectKind implements admission.Validator. -func (v *FakeValidator) GetObjectKind() schema.ObjectKind { return v } - -// DeepCopyObject implements admission.Validator. -func (v *FakeValidator) DeepCopyObject() runtime.Object { - return &FakeValidator{ErrorToReturn: v.ErrorToReturn, GVKToReturn: v.GVKToReturn} -} - -// GroupVersionKind implements admission.Validator. -func (v *FakeValidator) GroupVersionKind() schema.GroupVersionKind { - return v.GVKToReturn -} - -// SetGroupVersionKind implements admission.Validator. -func (v *FakeValidator) SetGroupVersionKind(gvk schema.GroupVersionKind) { - v.GVKToReturn = gvk -} diff --git a/pkg/webhook/admission/decode.go b/pkg/webhook/admission/decode.go index c7cb71b755..55f1cafb5e 100644 --- a/pkg/webhook/admission/decode.go +++ b/pkg/webhook/admission/decode.go @@ -19,7 +19,6 @@ package admission import ( "fmt" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/json" @@ -27,19 +26,35 @@ import ( // Decoder knows how to decode the contents of an admission // request into a concrete object. -type Decoder struct { +type Decoder interface { + // Decode decodes the inlined object in the AdmissionRequest into the passed-in runtime.Object. + // If you want decode the OldObject in the AdmissionRequest, use DecodeRaw. + // It errors out if req.Object.Raw is empty i.e. containing 0 raw bytes. + Decode(req Request, into runtime.Object) error + + // DecodeRaw decodes a RawExtension object into the passed-in runtime.Object. + // It errors out if rawObj is empty i.e. containing 0 raw bytes. + DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) error +} + +// decoder knows how to decode the contents of an admission +// request into a concrete object. +type decoder struct { codecs serializer.CodecFactory } -// NewDecoder creates a Decoder given the runtime.Scheme. -func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) { - return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil +// NewDecoder creates a decoder given the runtime.Scheme. +func NewDecoder(scheme *runtime.Scheme) Decoder { + if scheme == nil { + panic("scheme should never be nil") + } + return &decoder{codecs: serializer.NewCodecFactory(scheme)} } // Decode decodes the inlined object in the AdmissionRequest into the passed-in runtime.Object. // If you want decode the OldObject in the AdmissionRequest, use DecodeRaw. // It errors out if req.Object.Raw is empty i.e. containing 0 raw bytes. -func (d *Decoder) Decode(req Request, into runtime.Object) error { +func (d *decoder) Decode(req Request, into runtime.Object) error { // we error out if rawObj is an empty object. if len(req.Object.Raw) == 0 { return fmt.Errorf("there is no content to decode") @@ -49,7 +64,7 @@ func (d *Decoder) Decode(req Request, into runtime.Object) error { // DecodeRaw decodes a RawExtension object into the passed-in runtime.Object. // It errors out if rawObj is empty i.e. containing 0 raw bytes. -func (d *Decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) error { +func (d *decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) error { // NB(directxman12): there's a bug/weird interaction between decoders and // the API server where the API server doesn't send a GVK on the embedded // objects, which means the unstructured decoder refuses to decode. It @@ -62,9 +77,14 @@ func (d *Decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) er if len(rawObj.Raw) == 0 { return fmt.Errorf("there is no content to decode") } - if unstructuredInto, isUnstructured := into.(*unstructured.Unstructured); isUnstructured { + if unstructuredInto, isUnstructured := into.(runtime.Unstructured); isUnstructured { // unmarshal into unstructured's underlying object to avoid calling the decoder - return json.Unmarshal(rawObj.Raw, &unstructuredInto.Object) + var object map[string]interface{} + if err := json.Unmarshal(rawObj.Raw, &object); err != nil { + return err + } + unstructuredInto.SetUnstructuredContent(object) + return nil } deserializer := d.codecs.UniversalDeserializer() diff --git a/pkg/webhook/admission/decode_test.go b/pkg/webhook/admission/decode_test.go index c167c51026..130308800f 100644 --- a/pkg/webhook/admission/decode_test.go +++ b/pkg/webhook/admission/decode_test.go @@ -17,7 +17,7 @@ limitations under the License. package admission import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admission/v1" @@ -29,12 +29,10 @@ import ( ) var _ = Describe("Admission Webhook Decoder", func() { - var decoder *Decoder + var decoder Decoder BeforeEach(func() { By("creating a new decoder for a scheme") - var err error - decoder, err = NewDecoder(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) + decoder = NewDecoder(scheme.Scheme) Expect(decoder).NotTo(BeNil()) }) @@ -125,6 +123,8 @@ var _ = Describe("Admission Webhook Decoder", func() { })) }) + // NOTE: This will only pass if a GVK is provided. An unstructered object without a GVK may succeed + // in decoding to an alternate type. It("should fail to decode if the object in the request doesn't match the passed-in type", func() { By("trying to extract a pod from the quest into a node") Expect(decoder.Decode(req, &corev1.Node{})).NotTo(Succeed()) @@ -154,4 +154,35 @@ var _ = Describe("Admission Webhook Decoder", func() { "namespace": "default", })) }) + + req2 := Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: "CREATE", + Object: runtime.RawExtension{ + Raw: []byte(`{ + "metadata": { + "name": "foo", + "namespace": "default" + }, + "spec": { + "containers": [ + { + "image": "bar:v2", + "name": "bar" + } + ] + } + }`), + }, + OldObject: runtime.RawExtension{ + Object: nil, + }, + }, + } + + It("should decode a valid admission request without GVK", func() { + By("extracting the object from the request") + var target3 unstructured.Unstructured + Expect(decoder.DecodeRaw(req2.Object, &target3)).To(Succeed()) + }) }) diff --git a/pkg/webhook/admission/defaulter.go b/pkg/webhook/admission/defaulter.go deleted file mode 100644 index 0d9aa7a838..0000000000 --- a/pkg/webhook/admission/defaulter.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "context" - "encoding/json" - "net/http" - - "k8s.io/apimachinery/pkg/runtime" -) - -// Defaulter defines functions for setting defaults on resources. -type Defaulter interface { - runtime.Object - Default() -} - -// DefaultingWebhookFor creates a new Webhook for Defaulting the provided type. -func DefaultingWebhookFor(defaulter Defaulter) *Webhook { - return &Webhook{ - Handler: &mutatingHandler{defaulter: defaulter}, - } -} - -type mutatingHandler struct { - defaulter Defaulter - decoder *Decoder -} - -var _ DecoderInjector = &mutatingHandler{} - -// InjectDecoder injects the decoder into a mutatingHandler. -func (h *mutatingHandler) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil -} - -// Handle handles admission requests. -func (h *mutatingHandler) Handle(ctx context.Context, req Request) Response { - if h.defaulter == nil { - panic("defaulter should never be nil") - } - - // Get the object in the request - obj := h.defaulter.DeepCopyObject().(Defaulter) - if err := h.decoder.Decode(req, obj); err != nil { - return Errored(http.StatusBadRequest, err) - } - - // Default the object - obj.Default() - marshalled, err := json.Marshal(obj) - if err != nil { - return Errored(http.StatusInternalServerError, err) - } - - // Create the patch - return PatchResponseFromRaw(req.Object.Raw, marshalled) -} diff --git a/pkg/webhook/admission/defaulter_custom.go b/pkg/webhook/admission/defaulter_custom.go index a012784e43..a703cbd2c5 100644 --- a/pkg/webhook/admission/defaulter_custom.go +++ b/pkg/webhook/admission/defaulter_custom.go @@ -19,12 +19,16 @@ package admission import ( "context" "encoding/json" - "errors" "net/http" + "slices" + "gomodules.xyz/jsonpatch/v2" + admissionv1 "k8s.io/api/admission/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" ) // CustomDefaulter defines functions for setting defaults on resources. @@ -32,28 +36,48 @@ type CustomDefaulter interface { Default(ctx context.Context, obj runtime.Object) error } +type defaulterOptions struct { + removeUnknownOrOmitableFields bool +} + +// DefaulterOption defines the type of a CustomDefaulter's option +type DefaulterOption func(*defaulterOptions) + +// DefaulterRemoveUnknownOrOmitableFields makes the defaulter prune fields that are in the json object retrieved by the +// webhook but not in the local go type json representation. This happens for example when the CRD in the apiserver has +// fields that our go type doesn't know about, because it's outdated, or the field has a zero value and is `omitempty`. +func DefaulterRemoveUnknownOrOmitableFields(o *defaulterOptions) { + o.removeUnknownOrOmitableFields = true +} + // WithCustomDefaulter creates a new Webhook for a CustomDefaulter interface. -func WithCustomDefaulter(obj runtime.Object, defaulter CustomDefaulter) *Webhook { +func WithCustomDefaulter(scheme *runtime.Scheme, obj runtime.Object, defaulter CustomDefaulter, opts ...DefaulterOption) *Webhook { + options := &defaulterOptions{} + for _, o := range opts { + o(options) + } return &Webhook{ - Handler: &defaulterForType{object: obj, defaulter: defaulter}, + Handler: &defaulterForType{ + object: obj, + defaulter: defaulter, + decoder: NewDecoder(scheme), + removeUnknownOrOmitableFields: options.removeUnknownOrOmitableFields, + }, } } type defaulterForType struct { - defaulter CustomDefaulter - object runtime.Object - decoder *Decoder -} - -var _ DecoderInjector = &defaulterForType{} - -func (h *defaulterForType) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil + defaulter CustomDefaulter + object runtime.Object + decoder Decoder + removeUnknownOrOmitableFields bool } // Handle handles admission requests. func (h *defaulterForType) Handle(ctx context.Context, req Request) Response { + if h.decoder == nil { + panic("decoder should never be nil") + } if h.defaulter == nil { panic("defaulter should never be nil") } @@ -61,12 +85,30 @@ func (h *defaulterForType) Handle(ctx context.Context, req Request) Response { panic("object should never be nil") } + // Always skip when a DELETE operation received in custom mutation handler. + if req.Operation == admissionv1.Delete { + return Response{AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Result: &metav1.Status{ + Code: http.StatusOK, + }, + }} + } + + ctx = NewContextWithRequest(ctx, req) + // Get the object in the request obj := h.object.DeepCopyObject() if err := h.decoder.Decode(req, obj); err != nil { return Errored(http.StatusBadRequest, err) } + // Keep a copy of the object if needed + var originalObj runtime.Object + if !h.removeUnknownOrOmitableFields { + originalObj = obj.DeepCopyObject() + } + // Default the object if err := h.defaulter.Default(ctx, obj); err != nil { var apiStatus apierrors.APIStatus @@ -81,5 +123,43 @@ func (h *defaulterForType) Handle(ctx context.Context, req Request) Response { if err != nil { return Errored(http.StatusInternalServerError, err) } - return PatchResponseFromRaw(req.Object.Raw, marshalled) + + handlerResponse := PatchResponseFromRaw(req.Object.Raw, marshalled) + if !h.removeUnknownOrOmitableFields { + handlerResponse = h.dropSchemeRemovals(handlerResponse, originalObj, req.Object.Raw) + } + return handlerResponse +} + +func (h *defaulterForType) dropSchemeRemovals(r Response, original runtime.Object, raw []byte) Response { + const opRemove = "remove" + if !r.Allowed || r.PatchType == nil { + return r + } + + // If we don't have removals in the patch. + if !slices.ContainsFunc(r.Patches, func(o jsonpatch.JsonPatchOperation) bool { return o.Operation == opRemove }) { + return r + } + + // Get the raw to original patch + marshalledOriginal, err := json.Marshal(original) + if err != nil { + return Errored(http.StatusInternalServerError, err) + } + + patchOriginal, err := jsonpatch.CreatePatch(raw, marshalledOriginal) + if err != nil { + return Errored(http.StatusInternalServerError, err) + } + removedByScheme := sets.New(slices.DeleteFunc(patchOriginal, func(p jsonpatch.JsonPatchOperation) bool { return p.Operation != opRemove })...) + + r.Patches = slices.DeleteFunc(r.Patches, func(p jsonpatch.JsonPatchOperation) bool { + return p.Operation == opRemove && removedByScheme.Has(p) + }) + + if len(r.Patches) == 0 { + r.PatchType = nil + } + return r } diff --git a/pkg/webhook/admission/defaulter_custom_test.go b/pkg/webhook/admission/defaulter_custom_test.go new file mode 100644 index 0000000000..1bc26e59f4 --- /dev/null +++ b/pkg/webhook/admission/defaulter_custom_test.go @@ -0,0 +1,169 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "context" + "maps" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gomodules.xyz/jsonpatch/v2" + + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("Defaulter Handler", func() { + + It("should remove unknown fields when DefaulterRemoveUnknownFields is passed", func(ctx SpecContext) { + obj := &TestDefaulter{} + handler := WithCustomDefaulter(admissionScheme, obj, &TestCustomDefaulter{}, DefaulterRemoveUnknownOrOmitableFields) + + resp := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte(`{"newField":"foo", "totalReplicas":5}`), + }, + }, + }) + Expect(resp.Allowed).Should(BeTrue()) + Expect(resp.Patches).To(HaveLen(4)) + Expect(resp.Patches).To(ContainElements( + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/labels", + Value: map[string]any{"foo": "bar"}, + }, + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/replica", + Value: 2.0, + }, + jsonpatch.JsonPatchOperation{ + Operation: "remove", + Path: "/newField", + }, + jsonpatch.JsonPatchOperation{ + Operation: "remove", + Path: "/totalReplicas", + }, + )) + Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + + It("should preserve unknown fields by default", func(ctx SpecContext) { + obj := &TestDefaulter{} + handler := WithCustomDefaulter(admissionScheme, obj, &TestCustomDefaulter{}) + + resp := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte(`{"newField":"foo", "totalReplicas":5}`), + }, + }, + }) + Expect(resp.Allowed).Should(BeTrue()) + Expect(resp.Patches).To(HaveLen(3)) + Expect(resp.Patches).To(ContainElements( + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/labels", + Value: map[string]any{"foo": "bar"}, + }, + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/replica", + Value: 2.0, + }, + jsonpatch.JsonPatchOperation{ + Operation: "remove", + Path: "/totalReplicas", + }, + )) + Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + + It("should return ok if received delete verb in defaulter handler", func(ctx SpecContext) { + obj := &TestDefaulter{} + handler := WithCustomDefaulter(admissionScheme, obj, &TestCustomDefaulter{}) + resp := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + }, + }, + }) + Expect(resp.Allowed).Should(BeTrue()) + Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) + }) +}) + +// TestDefaulter. +var _ runtime.Object = &TestDefaulter{} + +type TestDefaulter struct { + Labels map[string]string `json:"labels,omitempty"` + + Replica int `json:"replica,omitempty"` + TotalReplicas int `json:"totalReplicas,omitempty"` +} + +var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestDefaulter"} + +func (d *TestDefaulter) GetObjectKind() schema.ObjectKind { return d } +func (d *TestDefaulter) DeepCopyObject() runtime.Object { + return &TestDefaulter{ + Labels: maps.Clone(d.Labels), + Replica: d.Replica, + TotalReplicas: d.TotalReplicas, + } +} + +func (d *TestDefaulter) GroupVersionKind() schema.GroupVersionKind { + return testDefaulterGVK +} + +func (d *TestDefaulter) SetGroupVersionKind(gvk schema.GroupVersionKind) {} + +var _ runtime.Object = &TestDefaulterList{} + +type TestDefaulterList struct{} + +func (*TestDefaulterList) GetObjectKind() schema.ObjectKind { return nil } +func (*TestDefaulterList) DeepCopyObject() runtime.Object { return nil } + +// TestCustomDefaulter +type TestCustomDefaulter struct{} + +func (d *TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) error { + o := obj.(*TestDefaulter) + + if o.Labels == nil { + o.Labels = map[string]string{} + } + o.Labels["foo"] = "bar" + + if o.Replica < 2 { + o.Replica = 2 + } + o.TotalReplicas = 0 + return nil +} diff --git a/pkg/webhook/admission/doc.go b/pkg/webhook/admission/doc.go index 0b274dd02b..8dc0cbec6f 100644 --- a/pkg/webhook/admission/doc.go +++ b/pkg/webhook/admission/doc.go @@ -20,9 +20,3 @@ Package admission provides implementation for admission webhook and methods to i See examples/mutatingwebhook.go and examples/validatingwebhook.go for examples of admission webhooks. */ package admission - -import ( - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" -) - -var log = logf.RuntimeLog.WithName("admission") diff --git a/pkg/webhook/admission/http.go b/pkg/webhook/admission/http.go index 3fa8872ff2..f049fb66e6 100644 --- a/pkg/webhook/admission/http.go +++ b/pkg/webhook/admission/http.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" v1 "k8s.io/api/admission/v1" @@ -35,6 +34,26 @@ import ( var admissionScheme = runtime.NewScheme() var admissionCodecs = serializer.NewCodecFactory(admissionScheme) +// adapted from https://github.com/kubernetes/kubernetes/blob/c28c2009181fcc44c5f6b47e10e62dacf53e4da0/staging/src/k8s.io/pod-security-admission/cmd/webhook/server/server.go +// +// From https://github.com/kubernetes/apiserver/blob/d6876a0600de06fef75968c4641c64d7da499f25/pkg/server/config.go#L433-L442C5: +// +// 1.5MB is the recommended client request size in byte +// the etcd server should accept. See +// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56. +// A request body might be encoded in json, and is converted to +// proto when persisted in etcd, so we allow 2x as the largest request +// body size to be accepted and decoded in a write request. +// +// For the admission request, we can infer that it contains at most two objects +// (the old and new versions of the object being admitted), each of which can +// be at most 3MB in size. For the rest of the request, we can assume that +// it will be less than 1MB in size. Therefore, we can set the max request +// size to 7MB. +// If your use case requires larger max request sizes, please +// open an issue (https://github.com/kubernetes-sigs/controller-runtime/issues/new). +const maxRequestSize = int64(7 * 1024 * 1024) + func init() { utilruntime.Must(v1.AddToScheme(admissionScheme)) utilruntime.Must(v1beta1.AddToScheme(admissionScheme)) @@ -43,36 +62,38 @@ func init() { var _ http.Handler = &Webhook{} func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var body []byte - var err error ctx := r.Context() if wh.WithContextFunc != nil { ctx = wh.WithContextFunc(ctx, r) } - var reviewResponse Response - if r.Body == nil { - err = errors.New("request body is empty") - wh.log.Error(err, "bad request") - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + if r.Body == nil || r.Body == http.NoBody { + err := errors.New("request body is empty") + wh.getLogger(nil).Error(err, "bad request") + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) return } defer r.Body.Close() - if body, err = ioutil.ReadAll(r.Body); err != nil { - wh.log.Error(err, "unable to read the body from the incoming request") - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + limitedReader := &io.LimitedReader{R: r.Body, N: maxRequestSize} + body, err := io.ReadAll(limitedReader) + if err != nil { + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request") + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) + return + } + if limitedReader.N <= 0 { + err := fmt.Errorf("request entity is too large; limit is %d bytes", maxRequestSize) + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request; limit reached") + wh.writeResponse(w, Errored(http.StatusRequestEntityTooLarge, err)) return } // verify the content type is accurate if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { err = fmt.Errorf("contentType=%s, expected application/json", contentType) - wh.log.Error(err, "unable to process a request with an unknown content type", "content type", contentType) - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + wh.getLogger(nil).Error(err, "unable to process a request with unknown content type") + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) return } @@ -89,15 +110,13 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { ar.SetGroupVersionKind(v1.SchemeGroupVersion.WithKind("AdmissionReview")) _, actualAdmRevGVK, err := admissionCodecs.UniversalDeserializer().Decode(body, nil, &ar) if err != nil { - wh.log.Error(err, "unable to decode the request") - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + wh.getLogger(nil).Error(err, "unable to decode the request") + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) return } - wh.log.V(1).Info("received request", "UID", req.UID, "kind", req.Kind, "resource", req.Resource) + wh.getLogger(&req).V(5).Info("received request") - reviewResponse = wh.Handle(ctx, req) - wh.writeResponseTyped(w, reviewResponse, actualAdmRevGVK) + wh.writeResponseTyped(w, wh.Handle(ctx, req), actualAdmRevGVK) } // writeResponse writes response to w generically, i.e. without encoding GVK information. @@ -125,15 +144,23 @@ func (wh *Webhook) writeResponseTyped(w io.Writer, response Response, admRevGVK // writeAdmissionResponse writes ar to w. func (wh *Webhook) writeAdmissionResponse(w io.Writer, ar v1.AdmissionReview) { if err := json.NewEncoder(w).Encode(ar); err != nil { - wh.log.Error(err, "unable to encode the response") - wh.writeResponse(w, Errored(http.StatusInternalServerError, err)) + wh.getLogger(nil).Error(err, "unable to encode and write the response") + // Since the `ar v1.AdmissionReview` is a clear and legal object, + // it should not have problem to be marshalled into bytes. + // The error here is probably caused by the abnormal HTTP connection, + // e.g., broken pipe, so we can only write the error response once, + // to avoid endless circular calling. + serverError := Errored(http.StatusInternalServerError, err) + if err = json.NewEncoder(w).Encode(v1.AdmissionReview{Response: &serverError.AdmissionResponse}); err != nil { + wh.getLogger(nil).Error(err, "still unable to encode and write the InternalServerError response") + } } else { res := ar.Response - if log := wh.log; log.V(1).Enabled() { + if log := wh.getLogger(nil); log.V(5).Enabled() { if res.Result != nil { - log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason) + log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason, "message", res.Result.Message) } - log.V(1).Info("wrote response", "UID", res.UID, "allowed", res.Allowed) + log.V(5).Info("wrote response", "requestID", res.UID, "allowed", res.Allowed) } } } diff --git a/pkg/webhook/admission/http_test.go b/pkg/webhook/admission/http_test.go index 7dd2d5bcfc..9cea9dd9e7 100644 --- a/pkg/webhook/admission/http_test.go +++ b/pkg/webhook/admission/http_test.go @@ -19,18 +19,17 @@ package admission import ( "bytes" "context" + "crypto/rand" "fmt" "io" "net/http" "net/http/httptest" + "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admission/v1" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("Admission Webhooks", func() { @@ -49,8 +48,6 @@ var _ = Describe("Admission Webhooks", func() { respRecorder = &httptest.ResponseRecorder{ Body: bytes.NewBuffer(nil), } - _, err := inject.LoggerInto(log.WithName("test-webhook"), webhook) - Expect(err).NotTo(HaveOccurred()) }) It("should return bad-request when given an empty body", func() { @@ -88,6 +85,32 @@ var _ = Describe("Admission Webhooks", func() { Expect(respRecorder.Body.String()).To(Equal(expected)) }) + It("should error when given a NoBody", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: http.NoBody, + } + + expected := `{"response":{"uid":"","allowed":false,"status":{"metadata":{},"message":"request body is empty","code":400}}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + + It("should error when given an infinite body", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: nopCloser{Reader: rand.Reader}, + } + + expected := `{"response":{"uid":"","allowed":false,"status":{"metadata":{},"message":"request entity is too large; limit is 7340032 bytes","code":413}}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + It("should return the response given by the handler with version defaulted to v1", func() { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, @@ -95,7 +118,6 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200}}} @@ -111,7 +133,6 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200}}} @@ -127,7 +148,6 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200}}} @@ -136,7 +156,7 @@ var _ = Describe("Admission Webhooks", func() { Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should present the Context from the HTTP request, if any", func() { + It("should present the Context from the HTTP request, if any", func(specCtx SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Body: nopCloser{Reader: bytes.NewBufferString(`{"request":{}}`)}, @@ -151,19 +171,18 @@ var _ = Describe("Admission Webhooks", func() { return Allowed(ctx.Value(key).(string)) }, }, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"reason":%q,"code":200}}} + expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"message":%q,"code":200}}} `, gvkJSONv1, value) - ctx, cancel := context.WithCancel(context.WithValue(context.Background(), key, value)) + ctx, cancel := context.WithCancel(context.WithValue(specCtx, key, value)) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should mutate the Context from the HTTP request, if func supplied", func() { + It("should mutate the Context from the HTTP request, if func supplied", func(specCtx SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Body: nopCloser{Reader: bytes.NewBufferString(`{"request":{}}`)}, @@ -179,17 +198,33 @@ var _ = Describe("Admission Webhooks", func() { WithContextFunc: func(ctx context.Context, r *http.Request) context.Context { return context.WithValue(ctx, key, r.Header["Content-Type"][0]) }, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"reason":%q,"code":200}}} + expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"message":%q,"code":200}}} `, gvkJSONv1, "application/json") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) }) + + It("should never run into circular calling if the writer has broken", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: nopCloser{Reader: bytes.NewBufferString(fmt.Sprintf(`{%s,"request":{}}`, gvkJSONv1))}, + } + webhook := &Webhook{ + Handler: &fakeHandler{}, + } + + bw := &brokenWriter{ResponseWriter: respRecorder} + Eventually(func() int { + // This should not be blocked by the circular calling of writeResponse and writeAdmissionResponse + webhook.ServeHTTP(bw, req) + return respRecorder.Body.Len() + }, time.Second*3).Should(Equal(0)) + }) }) }) @@ -200,20 +235,8 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } type fakeHandler struct { - invoked bool - fn func(context.Context, Request) Response - decoder *Decoder - injectedString string -} - -func (h *fakeHandler) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil -} - -func (h *fakeHandler) InjectString(s string) error { - h.injectedString = s - return nil + invoked bool + fn func(context.Context, Request) Response } func (h *fakeHandler) Handle(ctx context.Context, req Request) Response { @@ -225,3 +248,11 @@ func (h *fakeHandler) Handle(ctx context.Context, req Request) Response { Allowed: true, }} } + +type brokenWriter struct { + http.ResponseWriter +} + +func (bw *brokenWriter) Write(buf []byte) (int, error) { + return 0, fmt.Errorf("mock: write: broken pipe") +} diff --git a/pkg/webhook/admission/inject.go b/pkg/webhook/admission/inject.go deleted file mode 100644 index d5af0d598f..0000000000 --- a/pkg/webhook/admission/inject.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -// DecoderInjector is used by the ControllerManager to inject decoder into webhook handlers. -type DecoderInjector interface { - InjectDecoder(*Decoder) error -} - -// InjectDecoderInto will set decoder on i and return the result if it implements Decoder. Returns -// false if i does not implement Decoder. -func InjectDecoderInto(decoder *Decoder, i interface{}) (bool, error) { - if s, ok := i.(DecoderInjector); ok { - return true, s.InjectDecoder(decoder) - } - return false, nil -} diff --git a/pkg/webhook/admission/metrics/metrics.go b/pkg/webhook/admission/metrics/metrics.go new file mode 100644 index 0000000000..358a3a9162 --- /dev/null +++ b/pkg/webhook/admission/metrics/metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // WebhookPanics is a prometheus counter metrics which holds the total + // number of panics from webhooks. + WebhookPanics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "controller_runtime_webhook_panics_total", + Help: "Total number of webhook panics", + }, []string{}) +) + +func init() { + metrics.Registry.MustRegister( + WebhookPanics, + ) + // Init metric. + WebhookPanics.WithLabelValues().Add(0) +} diff --git a/pkg/webhook/admission/multi.go b/pkg/webhook/admission/multi.go index 26900cf2eb..ef9c456248 100644 --- a/pkg/webhook/admission/multi.go +++ b/pkg/webhook/admission/multi.go @@ -25,14 +25,13 @@ import ( jsonpatch "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) type multiMutating []Handler func (hs multiMutating) Handle(ctx context.Context, req Request) Response { patches := []jsonpatch.JsonPatchOperation{} + warnings := []string{} for _, handler := range hs { resp := handler.Handle(ctx, req) if !resp.Allowed { @@ -44,6 +43,7 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response { resp.PatchType, admissionv1.PatchTypeJSONPatch)) } patches = append(patches, resp.Patches...) + warnings = append(warnings, resp.Warnings...) } var err error marshaledPatch, err := json.Marshal(patches) @@ -57,36 +57,12 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response { Code: http.StatusOK, }, Patch: marshaledPatch, + Warnings: warnings, PatchType: func() *admissionv1.PatchType { pt := admissionv1.PatchTypeJSONPatch; return &pt }(), }, } } -// InjectFunc injects the field setter into the handlers. -func (hs multiMutating) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - for _, handler := range hs { - if err := f(handler); err != nil { - return err - } - } - - return nil -} - -// InjectDecoder injects the decoder into the handlers. -func (hs multiMutating) InjectDecoder(d *Decoder) error { - for _, handler := range hs { - if _, err := InjectDecoderInto(d, handler); err != nil { - return err - } - } - return nil -} - // MultiMutatingHandler combines multiple mutating webhook handlers into a single // mutating webhook handler. Handlers are called in sequential order, and the first // `allowed: false` response may short-circuit the rest. Users must take care to @@ -98,11 +74,13 @@ func MultiMutatingHandler(handlers ...Handler) Handler { type multiValidating []Handler func (hs multiValidating) Handle(ctx context.Context, req Request) Response { + warnings := []string{} for _, handler := range hs { resp := handler.Handle(ctx, req) if !resp.Allowed { return resp } + warnings = append(warnings, resp.Warnings...) } return Response{ AdmissionResponse: admissionv1.AdmissionResponse{ @@ -110,6 +88,7 @@ func (hs multiValidating) Handle(ctx context.Context, req Request) Response { Result: &metav1.Status{ Code: http.StatusOK, }, + Warnings: warnings, }, } } @@ -120,28 +99,3 @@ func (hs multiValidating) Handle(ctx context.Context, req Request) Response { func MultiValidatingHandler(handlers ...Handler) Handler { return multiValidating(handlers) } - -// InjectFunc injects the field setter into the handlers. -func (hs multiValidating) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - for _, handler := range hs { - if err := f(handler); err != nil { - return err - } - } - - return nil -} - -// InjectDecoder injects the decoder into the handlers. -func (hs multiValidating) InjectDecoder(d *Decoder) error { - for _, handler := range hs { - if _, err := InjectDecoderInto(d, handler); err != nil { - return err - } - } - return nil -} diff --git a/pkg/webhook/admission/multi_test.go b/pkg/webhook/admission/multi_test.go index a8b51872a2..888836ed67 100644 --- a/pkg/webhook/admission/multi_test.go +++ b/pkg/webhook/admission/multi_test.go @@ -19,7 +19,7 @@ package admission import ( "context" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" jsonpatch "gomodules.xyz/jsonpatch/v2" @@ -46,23 +46,46 @@ var _ = Describe("Multi-Handler Admission Webhooks", func() { }, } + withWarnings := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + return Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Warnings: []string{"handler-warning"}, + }, + } + }, + } + Context("with validating handlers", func() { - It("should deny the request if any handler denies the request", func() { + It("should deny the request if any handler denies the request", func(ctx SpecContext) { By("setting up a handler with accept and deny") handler := MultiValidatingHandler(alwaysAllow, alwaysDeny) By("checking that the handler denies the request") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeFalse()) + Expect(resp.Warnings).To(BeEmpty()) }) - It("should allow the request if all handlers allow the request", func() { + It("should allow the request if all handlers allow the request", func(ctx SpecContext) { By("setting up a handler with only accept") handler := MultiValidatingHandler(alwaysAllow, alwaysAllow) By("checking that the handler allows the request") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeTrue()) + Expect(resp.Warnings).To(BeEmpty()) + }) + + It("should show the warnings if all handlers allow the request", func(ctx SpecContext) { + By("setting up a handler with only accept") + handler := MultiValidatingHandler(alwaysAllow, withWarnings) + + By("checking that the handler allows the request") + resp := handler.Handle(ctx, Request{}) + Expect(resp.Allowed).To(BeTrue()) + Expect(resp.Warnings).To(HaveLen(1)) }) }) @@ -107,26 +130,60 @@ var _ = Describe("Multi-Handler Admission Webhooks", func() { }, } - It("should not return any patches if the request is denied", func() { + patcher3 := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + return Response{ + Patches: []jsonpatch.JsonPatchOperation{ + { + Operation: "add", + Path: "/metadata/annotation/newest-key", + Value: "value", + }, + }, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Warnings: []string{"annotation-warning"}, + PatchType: func() *admissionv1.PatchType { pt := admissionv1.PatchTypeJSONPatch; return &pt }(), + }, + } + }, + } + + It("should not return any patches if the request is denied", func(ctx SpecContext) { By("setting up a webhook with some patches and a deny") handler := MultiMutatingHandler(patcher1, patcher2, alwaysDeny) By("checking that the handler denies the request and produces no patches") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeFalse()) Expect(resp.Patches).To(BeEmpty()) }) - It("should produce all patches if the requests are all allowed", func() { + It("should produce all patches if the requests are all allowed", func(ctx SpecContext) { By("setting up a webhook with some patches") handler := MultiMutatingHandler(patcher1, patcher2, alwaysAllow) By("checking that the handler accepts the request and returns all patches") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeTrue()) Expect(resp.Patch).To(Equal([]byte( `[{"op":"add","path":"/metadata/annotation/new-key","value":"new-value"},` + `{"op":"replace","path":"/spec/replicas","value":"2"},{"op":"add","path":"/metadata/annotation/hello","value":"world"}]`))) }) + + It("should produce all patches if the requests are all allowed and show warnings", func(ctx SpecContext) { + By("setting up a webhook with some patches") + handler := MultiMutatingHandler(patcher1, patcher2, alwaysAllow, patcher3) + + By("checking that the handler accepts the request and returns all patches") + resp := handler.Handle(ctx, Request{}) + Expect(resp.Allowed).To(BeTrue()) + Expect(resp.Patch).To(Equal([]byte( + `[{"op":"add","path":"/metadata/annotation/new-key","value":"new-value"},` + + `{"op":"replace","path":"/spec/replicas","value":"2"},{"op":"add","path":"/metadata/annotation/hello","value":"world"},` + + `{"op":"add","path":"/metadata/annotation/newest-key","value":"value"}]`))) + Expect(resp.Warnings).To(HaveLen(1)) + }) + }) }) diff --git a/pkg/webhook/admission/response.go b/pkg/webhook/admission/response.go index 24ff1dee3c..ec1c88c989 100644 --- a/pkg/webhook/admission/response.go +++ b/pkg/webhook/admission/response.go @@ -26,21 +26,21 @@ import ( // Allowed constructs a response indicating that the given operation // is allowed (without any patches). -func Allowed(reason string) Response { - return ValidationResponse(true, reason) +func Allowed(message string) Response { + return ValidationResponse(true, message) } // Denied constructs a response indicating that the given operation // is not allowed. -func Denied(reason string) Response { - return ValidationResponse(false, reason) +func Denied(message string) Response { + return ValidationResponse(false, message) } // Patched constructs a response indicating that the given operation is // allowed, and that the target object should be modified by the given // JSONPatch operations. -func Patched(reason string, patches ...jsonpatch.JsonPatchOperation) Response { - resp := Allowed(reason) +func Patched(message string, patches ...jsonpatch.JsonPatchOperation) Response { + resp := Allowed(message) resp.Patches = patches return resp @@ -60,21 +60,24 @@ func Errored(code int32, err error) Response { } // ValidationResponse returns a response for admitting a request. -func ValidationResponse(allowed bool, reason string) Response { +func ValidationResponse(allowed bool, message string) Response { code := http.StatusForbidden + reason := metav1.StatusReasonForbidden if allowed { code = http.StatusOK + reason = "" } resp := Response{ AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: allowed, Result: &metav1.Status{ - Code: int32(code), + Code: int32(code), + Reason: reason, }, }, } - if len(reason) > 0 { - resp.Result.Reason = metav1.StatusReason(reason) + if len(message) > 0 { + resp.Result.Message = message } return resp } diff --git a/pkg/webhook/admission/response_test.go b/pkg/webhook/admission/response_test.go index e96b0e6ca7..03107c92f5 100644 --- a/pkg/webhook/admission/response_test.go +++ b/pkg/webhook/admission/response_test.go @@ -20,7 +20,7 @@ import ( "errors" "net/http" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" jsonpatch "gomodules.xyz/jsonpatch/v2" @@ -49,8 +49,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: true, Result: &metav1.Status{ - Code: http.StatusOK, - Reason: "acceptable", + Code: http.StatusOK, + Message: "acceptable", }, }, }, @@ -65,7 +65,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, }, }, }, @@ -78,8 +79,9 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, - Reason: "UNACCEPTABLE!", + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, + Message: "UNACCEPTABLE!", }, }, }, @@ -118,8 +120,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: true, Result: &metav1.Status{ - Code: http.StatusOK, - Reason: "some changes", + Code: http.StatusOK, + Message: "some changes", }, }, Patches: ops, @@ -146,15 +148,15 @@ var _ = Describe("Admission Webhook Response Helpers", func() { }) Describe("ValidationResponse", func() { - It("should populate a status with a reason when a reason is given", func() { + It("should populate a status with a message when a message is given", func() { By("checking that a message is populated for 'allowed' responses") Expect(ValidationResponse(true, "acceptable")).To(Equal( Response{ AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: true, Result: &metav1.Status{ - Code: http.StatusOK, - Reason: "acceptable", + Code: http.StatusOK, + Message: "acceptable", }, }, }, @@ -166,8 +168,9 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, - Reason: "UNACCEPTABLE!", + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, + Message: "UNACCEPTABLE!", }, }, }, @@ -193,7 +196,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, }, }, }, diff --git a/pkg/webhook/admission/validator.go b/pkg/webhook/admission/validator.go deleted file mode 100644 index 4b27e75ede..0000000000 --- a/pkg/webhook/admission/validator.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "context" - goerrors "errors" - "net/http" - - v1 "k8s.io/api/admission/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" -) - -// Validator defines functions for validating an operation. -type Validator interface { - runtime.Object - ValidateCreate() error - ValidateUpdate(old runtime.Object) error - ValidateDelete() error -} - -// ValidatingWebhookFor creates a new Webhook for validating the provided type. -func ValidatingWebhookFor(validator Validator) *Webhook { - return &Webhook{ - Handler: &validatingHandler{validator: validator}, - } -} - -type validatingHandler struct { - validator Validator - decoder *Decoder -} - -var _ DecoderInjector = &validatingHandler{} - -// InjectDecoder injects the decoder into a validatingHandler. -func (h *validatingHandler) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil -} - -// Handle handles admission requests. -func (h *validatingHandler) Handle(ctx context.Context, req Request) Response { - if h.validator == nil { - panic("validator should never be nil") - } - - // Get the object in the request - obj := h.validator.DeepCopyObject().(Validator) - if req.Operation == v1.Create { - err := h.decoder.Decode(req, obj) - if err != nil { - return Errored(http.StatusBadRequest, err) - } - - err = obj.ValidateCreate() - if err != nil { - var apiStatus apierrors.APIStatus - if goerrors.As(err, &apiStatus) { - return validationResponseFromStatus(false, apiStatus.Status()) - } - return Denied(err.Error()) - } - } - - if req.Operation == v1.Update { - oldObj := obj.DeepCopyObject() - - err := h.decoder.DecodeRaw(req.Object, obj) - if err != nil { - return Errored(http.StatusBadRequest, err) - } - err = h.decoder.DecodeRaw(req.OldObject, oldObj) - if err != nil { - return Errored(http.StatusBadRequest, err) - } - - err = obj.ValidateUpdate(oldObj) - if err != nil { - var apiStatus apierrors.APIStatus - if goerrors.As(err, &apiStatus) { - return validationResponseFromStatus(false, apiStatus.Status()) - } - return Denied(err.Error()) - } - } - - if req.Operation == v1.Delete { - // In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346 - // OldObject contains the object being deleted - err := h.decoder.DecodeRaw(req.OldObject, obj) - if err != nil { - return Errored(http.StatusBadRequest, err) - } - - err = obj.ValidateDelete() - if err != nil { - var apiStatus apierrors.APIStatus - if goerrors.As(err, &apiStatus) { - return validationResponseFromStatus(false, apiStatus.Status()) - } - return Denied(err.Error()) - } - } - - return Allowed("") -} diff --git a/pkg/webhook/admission/validator_custom.go b/pkg/webhook/admission/validator_custom.go index 38d5565111..ef1be52a8f 100644 --- a/pkg/webhook/admission/validator_custom.go +++ b/pkg/webhook/admission/validator_custom.go @@ -27,36 +27,46 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// Warnings represents warning messages. +type Warnings []string + // CustomValidator defines functions for validating an operation. +// The object to be validated is passed into methods as a parameter. type CustomValidator interface { - ValidateCreate(ctx context.Context, obj runtime.Object) error - ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error - ValidateDelete(ctx context.Context, obj runtime.Object) error + // ValidateCreate validates the object on creation. + // The optional warnings will be added to the response as warning messages. + // Return an error if the object is invalid. + ValidateCreate(ctx context.Context, obj runtime.Object) (warnings Warnings, err error) + + // ValidateUpdate validates the object on update. + // The optional warnings will be added to the response as warning messages. + // Return an error if the object is invalid. + ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (warnings Warnings, err error) + + // ValidateDelete validates the object on deletion. + // The optional warnings will be added to the response as warning messages. + // Return an error if the object is invalid. + ValidateDelete(ctx context.Context, obj runtime.Object) (warnings Warnings, err error) } // WithCustomValidator creates a new Webhook for validating the provided type. -func WithCustomValidator(obj runtime.Object, validator CustomValidator) *Webhook { +func WithCustomValidator(scheme *runtime.Scheme, obj runtime.Object, validator CustomValidator) *Webhook { return &Webhook{ - Handler: &validatorForType{object: obj, validator: validator}, + Handler: &validatorForType{object: obj, validator: validator, decoder: NewDecoder(scheme)}, } } type validatorForType struct { validator CustomValidator object runtime.Object - decoder *Decoder -} - -var _ DecoderInjector = &validatorForType{} - -// InjectDecoder injects the decoder into a validatingHandler. -func (h *validatorForType) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil + decoder Decoder } // Handle handles admission requests. func (h *validatorForType) Handle(ctx context.Context, req Request) Response { + if h.decoder == nil { + panic("decoder should never be nil") + } if h.validator == nil { panic("validator should never be nil") } @@ -64,17 +74,24 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response { panic("object should never be nil") } + ctx = NewContextWithRequest(ctx, req) + // Get the object in the request obj := h.object.DeepCopyObject() var err error + var warnings []string + switch req.Operation { + case v1.Connect: + // No validation for connect requests. + // TODO(vincepri): Should we validate CONNECT requests? In what cases? case v1.Create: if err := h.decoder.Decode(req, obj); err != nil { return Errored(http.StatusBadRequest, err) } - err = h.validator.ValidateCreate(ctx, obj) + warnings, err = h.validator.ValidateCreate(ctx, obj) case v1.Update: oldObj := obj.DeepCopyObject() if err := h.decoder.DecodeRaw(req.Object, obj); err != nil { @@ -84,7 +101,7 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response { return Errored(http.StatusBadRequest, err) } - err = h.validator.ValidateUpdate(ctx, oldObj, obj) + warnings, err = h.validator.ValidateUpdate(ctx, oldObj, obj) case v1.Delete: // In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346 // OldObject contains the object being deleted @@ -92,20 +109,20 @@ func (h *validatorForType) Handle(ctx context.Context, req Request) Response { return Errored(http.StatusBadRequest, err) } - err = h.validator.ValidateDelete(ctx, obj) + warnings, err = h.validator.ValidateDelete(ctx, obj) default: - return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation request %q", req.Operation)) + return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation %q", req.Operation)) } // Check the error message first. if err != nil { var apiStatus apierrors.APIStatus if errors.As(err, &apiStatus) { - return validationResponseFromStatus(false, apiStatus.Status()) + return validationResponseFromStatus(false, apiStatus.Status()).WithWarnings(warnings...) } - return Denied(err.Error()) + return Denied(err.Error()).WithWarnings(warnings...) } // Return allowed if everything succeeded. - return Allowed("") + return Allowed("").WithWarnings(warnings...) } diff --git a/pkg/webhook/admission/validator_custom_test.go b/pkg/webhook/admission/validator_custom_test.go new file mode 100644 index 0000000000..7c9615df71 --- /dev/null +++ b/pkg/webhook/admission/validator_custom_test.go @@ -0,0 +1,508 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "context" + "errors" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + admissionv1 "k8s.io/api/admission/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var fakeValidatorVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "fakeValidator"} + +var _ = Describe("customValidatingHandler", func() { + + Context("when dealing with successful results without warning", func() { + val := &fakeCustomValidator{ErrorToReturn: nil, GVKToReturn: fakeValidatorVK, WarningsToReturn: nil} + f := &fakeValidator{} + handler := WithCustomValidator(admissionScheme, f, val) + + It("should return 200 in response when create succeeds", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeTrue()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + + It("should return 200 in response when update succeeds", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeTrue()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + + It("should return 200 in response when delete succeeds", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeTrue()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + }) + + const warningMessage = "warning message" + const anotherWarningMessage = "another warning message" + Context("when dealing with successful results with warning", func() { + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: nil, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{ + warningMessage, + anotherWarningMessage, + }} + handler := WithCustomValidator(admissionScheme, f, val) + It("should return 200 in response when create succeeds, with warning messages", func(ctx SpecContext) { + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeTrue()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) + }) + + It("should return 200 in response when update succeeds, with warning messages", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeTrue()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) + }) + + It("should return 200 in response when delete succeeds, with warning messages", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeTrue()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) + }) + }) + + Context("when dealing with Status errors, with warning messages", func() { + // Status error would overwrite the warning messages, so no warning messages should be observed. + expectedError := &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Message: "some message", + Code: http.StatusUnprocessableEntity, + }, + } + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{warningMessage, anotherWarningMessage}} + handler := WithCustomValidator(admissionScheme, f, val) + + It("should propagate the Status from ValidateCreate's return value to the HTTP response", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) + Expect(*response.Result).Should(Equal(expectedError.Status())) + Expect(response.AdmissionResponse.Warnings).Should(ContainElements(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElements(anotherWarningMessage)) + + }) + + It("should propagate the Status from ValidateUpdate's return value to the HTTP response", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) + Expect(*response.Result).Should(Equal(expectedError.Status())) + Expect(response.AdmissionResponse.Warnings).Should(ContainElements(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElements(anotherWarningMessage)) + + }) + + It("should propagate the Status from ValidateDelete's return value to the HTTP response", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) + Expect(*response.Result).Should(Equal(expectedError.Status())) + Expect(response.AdmissionResponse.Warnings).Should(ContainElements(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElements(anotherWarningMessage)) + + }) + + }) + + Context("when dealing with Status errors, without warning messages", func() { + + expectedError := &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Message: "some message", + Code: http.StatusUnprocessableEntity, + }, + } + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: nil} + handler := WithCustomValidator(admissionScheme, f, val) + + It("should propagate the Status from ValidateCreate's return value to the HTTP response", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) + Expect(*response.Result).Should(Equal(expectedError.Status())) + + }) + + It("should propagate the Status from ValidateUpdate's return value to the HTTP response", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) + Expect(*response.Result).Should(Equal(expectedError.Status())) + + }) + + It("should propagate the Status from ValidateDelete's return value to the HTTP response", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) + + Expect(*response.Result).Should(Equal(expectedError.Status())) + + }) + + }) + + Context("when dealing with non-status errors, without warning messages", func() { + + expectedError := errors.New("some error") + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK} + handler := WithCustomValidator(admissionScheme, f, val) + + It("should return 403 response when ValidateCreate with error message embedded", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) + + }) + + It("should return 403 response when ValidateUpdate returns non-APIStatus error", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) + + }) + + It("should return 403 response when ValidateDelete returns non-APIStatus error", func(ctx SpecContext) { + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) + }) + }) + + Context("when dealing with non-status errors, with warning messages", func() { + + expectedError := errors.New("some error") + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{warningMessage, anotherWarningMessage}} + handler := WithCustomValidator(admissionScheme, f, val) + + It("should return 403 response when ValidateCreate with error message embedded", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeFalse()) + + Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) + }) + + It("should return 403 response when ValidateUpdate returns non-APIStatus error", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Update, + Object: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + OldObject: runtime.RawExtension{ + + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) + + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) + + }) + + It("should return 403 response when ValidateDelete returns non-APIStatus error", func(ctx SpecContext) { + response := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + Object: f, + }, + }, + }) + Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(warningMessage)) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) + + }) + }) + + PIt("should return 400 in response when create fails on decode", func() {}) + + PIt("should return 400 in response when update fails on decoding new object", func() {}) + + PIt("should return 400 in response when update fails on decoding old object", func() {}) + + PIt("should return 400 in response when delete fails on decode", func() {}) + +}) + +// fakeCustomValidator provides fake validating webhook functionality for testing +// It implements the admission.CustomValidator interface and +// rejects all requests with the same configured error +// or passes if ErrorToReturn is nil. +// And it would always return configured warning messages WarningsToReturn. +type fakeCustomValidator struct { + // ErrorToReturn is the error for which the fakeValidator rejects all requests + ErrorToReturn error `json:"errorToReturn,omitempty"` + // GVKToReturn is the GroupVersionKind that the webhook operates on + GVKToReturn schema.GroupVersionKind + // WarningsToReturn is the warnings for fakeValidator returns to all requests + WarningsToReturn []string +} + +func (v *fakeCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (warnings Warnings, err error) { + return v.WarningsToReturn, v.ErrorToReturn +} + +func (v *fakeCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (warnings Warnings, err error) { + return v.WarningsToReturn, v.ErrorToReturn +} + +func (v *fakeCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (warnings Warnings, err error) { + return v.WarningsToReturn, v.ErrorToReturn +} + +type fakeValidator struct { + // GVKToReturn is the GroupVersionKind that the webhook operates on + GVKToReturn schema.GroupVersionKind +} + +func (v *fakeValidator) SetGroupVersionKind(gvk schema.GroupVersionKind) { + v.GVKToReturn = gvk +} + +func (v *fakeValidator) GroupVersionKind() schema.GroupVersionKind { + return v.GVKToReturn +} + +func (v *fakeValidator) GetObjectKind() schema.ObjectKind { + return v +} + +func (v *fakeValidator) DeepCopyObject() runtime.Object { + return &fakeValidator{ + GVKToReturn: v.GVKToReturn, + } +} diff --git a/pkg/webhook/admission/validator_test.go b/pkg/webhook/admission/validator_test.go deleted file mode 100644 index 7fe19268d9..0000000000 --- a/pkg/webhook/admission/validator_test.go +++ /dev/null @@ -1,240 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "context" - goerrors "errors" - "net/http" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission/admissiontest" - - admissionv1 "k8s.io/api/admission/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/kubernetes/scheme" -) - -var fakeValidatorVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "fakeValidator"} - -var _ = Describe("validatingHandler", func() { - - decoder, _ := NewDecoder(scheme.Scheme) - - Context("when dealing with successful results", func() { - - f := &admissiontest.FakeValidator{ErrorToReturn: nil, GVKToReturn: fakeValidatorVK} - handler := validatingHandler{validator: f, decoder: decoder} - - It("should return 200 in response when create succeeds", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Create, - Object: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - - Expect(response.Allowed).Should(BeTrue()) - Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) - }) - - It("should return 200 in response when update succeeds", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Update, - Object: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - Expect(response.Allowed).Should(BeTrue()) - Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) - }) - - It("should return 200 in response when delete succeeds", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Delete, - OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - Expect(response.Allowed).Should(BeTrue()) - Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) - }) - - }) - - Context("when dealing with Status errors", func() { - - expectedError := &apierrors.StatusError{ - ErrStatus: metav1.Status{ - Message: "some message", - Code: http.StatusUnprocessableEntity, - }, - } - f := &admissiontest.FakeValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK} - handler := validatingHandler{validator: f, decoder: decoder} - - It("should propagate the Status from ValidateCreate's return value to the HTTP response", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Create, - Object: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - - Expect(response.Allowed).Should(BeFalse()) - Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) - Expect(*response.Result).Should(Equal(expectedError.Status())) - - }) - - It("should propagate the Status from ValidateUpdate's return value to the HTTP response", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Update, - Object: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - - Expect(response.Allowed).Should(BeFalse()) - Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) - Expect(*response.Result).Should(Equal(expectedError.Status())) - - }) - - It("should propagate the Status from ValidateDelete's return value to the HTTP response", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Delete, - OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - - Expect(response.Allowed).Should(BeFalse()) - Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) - Expect(*response.Result).Should(Equal(expectedError.Status())) - - }) - - }) - Context("when dealing with non-status errors", func() { - - expectedError := goerrors.New("some error") - f := &admissiontest.FakeValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK} - handler := validatingHandler{validator: f, decoder: decoder} - - It("should return 403 response when ValidateCreate with error message embedded", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Create, - Object: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - Expect(response.Allowed).Should(BeFalse()) - Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) - Expect(string(response.Result.Reason)).Should(Equal(expectedError.Error())) - - }) - - It("should return 403 response when ValidateUpdate returns non-APIStatus error", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Update, - Object: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - Expect(response.Allowed).Should(BeFalse()) - Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) - Expect(string(response.Result.Reason)).Should(Equal(expectedError.Error())) - - }) - - It("should return 403 response when ValidateDelete returns non-APIStatus error", func() { - - response := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Delete, - OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, - }, - }, - }) - Expect(response.Allowed).Should(BeFalse()) - Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) - Expect(string(response.Result.Reason)).Should(Equal(expectedError.Error())) - - }) - - }) - - PIt("should return 400 in response when create fails on decode", func() {}) - - PIt("should return 400 in response when update fails on decoding new object", func() {}) - - PIt("should return 400 in response when update fails on decoding old object", func() {}) - - PIt("should return 400 in response when delete fails on decode", func() {}) - -}) diff --git a/pkg/webhook/admission/webhook.go b/pkg/webhook/admission/webhook.go index cf7dbcf68d..cba6da2cb0 100644 --- a/pkg/webhook/admission/webhook.go +++ b/pkg/webhook/admission/webhook.go @@ -19,18 +19,20 @@ package admission import ( "context" "errors" + "fmt" "net/http" + "sync" "github.com/go-logr/logr" - jsonpatch "gomodules.xyz/jsonpatch/v2" + "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/json" - "k8s.io/client-go/kubernetes/scheme" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" + admissionmetrics "sigs.k8s.io/controller-runtime/pkg/webhook/admission/metrics" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" ) @@ -121,101 +123,102 @@ type Webhook struct { // and potentially patches to apply to the handler. Handler Handler + // RecoverPanic indicates whether the panic caused by webhook should be recovered. + // Defaults to true. + RecoverPanic *bool + // WithContextFunc will allow you to take the http.Request.Context() and // add any additional information such as passing the request path or // headers thus allowing you to read them from within the handler WithContextFunc func(context.Context, *http.Request) context.Context - // decoder is constructed on receiving a scheme and passed down to then handler - decoder *Decoder + // LogConstructor is used to construct a logger for logging messages during webhook calls + // based on the given base logger (which might carry more values like the webhook's path). + // Note: LogConstructor has to be able to handle nil requests as we are also using it + // outside the context of requests. + LogConstructor func(base logr.Logger, req *Request) logr.Logger - log logr.Logger + setupLogOnce sync.Once + log logr.Logger } -// InjectLogger gets a handle to a logging instance, hopefully with more info about this particular webhook. -func (wh *Webhook) InjectLogger(l logr.Logger) error { - wh.log = l - return nil +// WithRecoverPanic takes a bool flag which indicates whether the panic caused by webhook should be recovered. +// Defaults to true. +func (wh *Webhook) WithRecoverPanic(recoverPanic bool) *Webhook { + wh.RecoverPanic = &recoverPanic + return wh } // Handle processes AdmissionRequest. // If the webhook is mutating type, it delegates the AdmissionRequest to each handler and merge the patches. // If the webhook is validating type, it delegates the AdmissionRequest to each handler and // deny the request if anyone denies. -func (wh *Webhook) Handle(ctx context.Context, req Request) Response { +func (wh *Webhook) Handle(ctx context.Context, req Request) (response Response) { + defer func() { + if r := recover(); r != nil { + admissionmetrics.WebhookPanics.WithLabelValues().Inc() + + if wh.RecoverPanic == nil || *wh.RecoverPanic { + for _, fn := range utilruntime.PanicHandlers { + fn(ctx, r) + } + response = Errored(http.StatusInternalServerError, fmt.Errorf("panic: %v [recovered]", r)) + // Note: We explicitly have to set the response UID. Usually that is done via resp.Complete below, + // but if we encounter a panic in wh.Handler.Handle we are never going to reach resp.Complete. + response.UID = req.UID + return + } + + log := logf.FromContext(ctx) + log.Info(fmt.Sprintf("Observed a panic in webhook: %v", r)) + panic(r) + } + }() + + reqLog := wh.getLogger(&req) + ctx = logf.IntoContext(ctx, reqLog) + resp := wh.Handler.Handle(ctx, req) if err := resp.Complete(req); err != nil { - wh.log.Error(err, "unable to encode response") - return Errored(http.StatusInternalServerError, errUnableToEncodeResponse) + reqLog.Error(err, "unable to encode response") + resp := Errored(http.StatusInternalServerError, errUnableToEncodeResponse) + // Note: We explicitly have to set the response UID. Usually that is done via resp.Complete. + resp.UID = req.UID + return resp } return resp } -// InjectScheme injects a scheme into the webhook, in order to construct a Decoder. -func (wh *Webhook) InjectScheme(s *runtime.Scheme) error { - // TODO(directxman12): we should have a better way to pass this down - - var err error - wh.decoder, err = NewDecoder(s) - if err != nil { - return err - } - - // inject the decoder here too, just in case the order of calling this is not - // scheme first, then inject func - if wh.Handler != nil { - if _, err := InjectDecoderInto(wh.GetDecoder(), wh.Handler); err != nil { - return err +// getLogger constructs a logger from the injected log and LogConstructor. +func (wh *Webhook) getLogger(req *Request) logr.Logger { + wh.setupLogOnce.Do(func() { + if wh.log.GetSink() == nil { + wh.log = logf.Log.WithName("admission") } - } + }) - return nil -} - -// GetDecoder returns a decoder to decode the objects embedded in admission requests. -// It may be nil if we haven't received a scheme to use to determine object types yet. -func (wh *Webhook) GetDecoder() *Decoder { - return wh.decoder + logConstructor := wh.LogConstructor + if logConstructor == nil { + logConstructor = DefaultLogConstructor + } + return logConstructor(wh.log, req) } -// InjectFunc injects the field setter into the webhook. -func (wh *Webhook) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - - // also inject a decoder, and wrap this so that we get a setFields - // that injects a decoder (hopefully things don't ignore the duplicate - // InjectorInto call). - - var setFields inject.Func - setFields = func(target interface{}) error { - if err := f(target); err != nil { - return err - } - - if _, err := inject.InjectorInto(setFields, target); err != nil { - return err - } - - if _, err := InjectDecoderInto(wh.GetDecoder(), target); err != nil { - return err - } - - return nil +// DefaultLogConstructor adds some commonly interesting fields to the given logger. +func DefaultLogConstructor(base logr.Logger, req *Request) logr.Logger { + if req != nil { + return base.WithValues("object", klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + "resource", req.Resource, "user", req.UserInfo.Username, + "requestID", req.UID, + ) } - - return setFields(wh.Handler) + return base } // StandaloneOptions let you configure a StandaloneWebhook. type StandaloneOptions struct { - // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources - // Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better - // idea to pass your own scheme in. See the documentation in pkg/scheme for more information. - Scheme *runtime.Scheme // Logger to be used by the webhook. // If none is set, it defaults to log.Log global logger. Logger logr.Logger @@ -235,21 +238,29 @@ type StandaloneOptions struct { // in your own server/mux. In order to be accessed by a kubernetes cluster, // all webhook servers require TLS. func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, error) { - if opts.Scheme == nil { - opts.Scheme = scheme.Scheme - } - - if err := hook.InjectScheme(opts.Scheme); err != nil { - return nil, err - } - - if opts.Logger == nil { - opts.Logger = logf.RuntimeLog.WithName("webhook") + if opts.Logger.GetSink() != nil { + hook.log = opts.Logger } - hook.log = opts.Logger - if opts.MetricsPath == "" { return hook, nil } return metrics.InstrumentedHook(opts.MetricsPath, hook), nil } + +// requestContextKey is how we find the admission.Request in a context.Context. +type requestContextKey struct{} + +// RequestFromContext returns an admission.Request from ctx. +func RequestFromContext(ctx context.Context) (Request, error) { + if v, ok := ctx.Value(requestContextKey{}).(Request); ok { + return v, nil + } + + return Request{}, errors.New("admission.Request not found in context") +} + +// NewContextWithRequest returns a new Context, derived from ctx, which carries the +// provided admission.Request. +func NewContextWithRequest(ctx context.Context, req Request) context.Context { + return context.WithValue(ctx, requestContextKey{}, req) +} diff --git a/pkg/webhook/admission/webhook_test.go b/pkg/webhook/admission/webhook_test.go index 73b0be1694..5176077368 100644 --- a/pkg/webhook/admission/webhook_test.go +++ b/pkg/webhook/admission/webhook_test.go @@ -18,22 +18,35 @@ package admission import ( "context" + "io" "net/http" - . "github.com/onsi/ginkgo" + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - jsonpatch "gomodules.xyz/jsonpatch/v2" + "github.com/onsi/gomega/gbytes" + "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" + authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" machinerytypes "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" ) var _ = Describe("Admission Webhooks", func() { + var ( + logBuffer *gbytes.Buffer + testLogger logr.Logger + ) + + BeforeEach(func() { + logBuffer = gbytes.NewBuffer() + testLogger = zap.New(zap.JSONEncoder(), zap.WriteTo(io.MultiWriter(logBuffer, GinkgoWriter))) + }) + allowHandler := func() *Webhook { handler := &fakeHandler{ fn: func(ctx context.Context, req Request) Response { @@ -46,46 +59,45 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), } return webhook } - It("should invoke the handler to get a response", func() { + It("should invoke the handler to get a response", func(ctx SpecContext) { By("setting up a webhook with an allow handler") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that it allowed the request") Expect(resp.Allowed).To(BeTrue()) }) - It("should ensure that the response's UID is set to the request's UID", func() { + It("should ensure that the response's UID is set to the request's UID", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{AdmissionRequest: admissionv1.AdmissionRequest{UID: "foobar"}}) + resp := webhook.Handle(ctx, Request{AdmissionRequest: admissionv1.AdmissionRequest{UID: "foobar"}}) By("checking that the response share's the request's UID") Expect(resp.UID).To(Equal(machinerytypes.UID("foobar"))) }) - It("should populate the status on a response if one is not provided", func() { + It("should populate the status on a response if one is not provided", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the response share's the request's UID") Expect(resp.Result).To(Equal(&metav1.Status{Code: http.StatusOK})) }) - It("shouldn't overwrite the status on a response", func() { + It("shouldn't overwrite the status on a response", func(ctx SpecContext) { By("setting up a webhook that sets a status") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { @@ -96,28 +108,26 @@ var _ = Describe("Admission Webhooks", func() { }, } }), - log: logf.RuntimeLog.WithName("webhook"), } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the message is intact") Expect(resp.Result).NotTo(BeNil()) Expect(resp.Result.Message).To(Equal("Ground Control to Major Tom")) }) - It("should serialize patch operations into a single jsonpatch blob", func() { + It("should serialize patch operations into a single jsonpatch blob", func(ctx SpecContext) { By("setting up a webhook with a patching handler") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { return Patched("", jsonpatch.Operation{Operation: "add", Path: "/a", Value: 2}, jsonpatch.Operation{Operation: "replace", Path: "/b", Value: 4}) }), - log: logf.RuntimeLog.WithName("webhook"), } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that a JSON patch is populated on the response") patchType := admissionv1.PatchTypeJSONPatch @@ -125,93 +135,162 @@ var _ = Describe("Admission Webhooks", func() { Expect(resp.Patch).To(Equal([]byte(`[{"op":"add","path":"/a","value":2},{"op":"replace","path":"/b","value":4}]`))) }) - Describe("dependency injection", func() { - It("should set dependencies passed in on the handler", func() { - By("setting up a webhook and injecting it with a injection func that injects a string") - setFields := func(target interface{}) error { - inj, ok := target.(stringInjector) - if !ok { - return nil + It("should pass a request logger via the context", func(ctx SpecContext) { + By("setting up a webhook that uses the request logger") + webhook := &Webhook{ + Handler: HandlerFunc(func(ctx context.Context, req Request) Response { + logf.FromContext(ctx).Info("Received request") + + return Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + }, } + }), + log: testLogger, + } - return inj.InjectString("something") - } - handler := &fakeHandler{} - webhook := &Webhook{ - Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), + By("invoking the webhook") + resp := webhook.Handle(ctx, Request{AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "test123", + Name: "foo", + Namespace: "bar", + Resource: metav1.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + UserInfo: authenticationv1.UserInfo{ + Username: "tim", + }, + }}) + Expect(resp.Allowed).To(BeTrue()) + + By("checking that the log message contains the request fields") + Eventually(logBuffer).Should(gbytes.Say(`"msg":"Received request","object":{"name":"foo","namespace":"bar"},"namespace":"bar","name":"foo","resource":{"group":"apps","version":"v1","resource":"deployments"},"user":"tim","requestID":"test123"}`)) + }) + + It("should pass a request logger created by LogConstructor via the context", func(ctx SpecContext) { + By("setting up a webhook that uses the request logger") + webhook := &Webhook{ + Handler: HandlerFunc(func(ctx context.Context, req Request) Response { + logf.FromContext(ctx).Info("Received request") + + return Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + }, + } + }), + LogConstructor: func(base logr.Logger, req *Request) logr.Logger { + return base.WithValues("operation", req.Operation, "requestID", req.UID) + }, + log: testLogger, + } + + By("invoking the webhook") + resp := webhook.Handle(ctx, Request{AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "test123", + Operation: admissionv1.Create, + }}) + Expect(resp.Allowed).To(BeTrue()) + + By("checking that the log message contains the request fields") + Eventually(logBuffer).Should(gbytes.Say(`"msg":"Received request","operation":"CREATE","requestID":"test123"}`)) + }) + + Describe("panic recovery", func() { + It("should recover panic if RecoverPanic is true by default", func(ctx SpecContext) { + panicHandler := func() *Webhook { + handler := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + panic("fake panic test") + }, + } + webhook := &Webhook{ + Handler: handler, + // RecoverPanic defaults to true. + } + + return webhook } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) - By("checking that the string was injected") - Expect(handler.injectedString).To(Equal("something")) + By("setting up a webhook with a panicking handler") + webhook := panicHandler() + + By("invoking the webhook") + resp := webhook.Handle(ctx, Request{}) + + By("checking that it errored the request") + Expect(resp.Allowed).To(BeFalse()) + Expect(resp.Result.Code).To(Equal(int32(http.StatusInternalServerError))) + Expect(resp.Result.Message).To(Equal("panic: fake panic test [recovered]")) }) - It("should inject a decoder into the handler", func() { - By("setting up a webhook and injecting it with a injection func that injects a scheme") - setFields := func(target interface{}) error { - if _, err := inject.SchemeInto(runtime.NewScheme(), target); err != nil { - return err + It("should recover panic if RecoverPanic is true", func(ctx SpecContext) { + panicHandler := func() *Webhook { + handler := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + panic("fake panic test") + }, } - return nil - } - handler := &fakeHandler{} - webhook := &Webhook{ - Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), + webhook := &Webhook{ + Handler: handler, + RecoverPanic: ptr.To[bool](true), + } + + return webhook } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) - By("checking that the decoder was injected") - Expect(handler.decoder).NotTo(BeNil()) + By("setting up a webhook with a panicking handler") + webhook := panicHandler() + + By("invoking the webhook") + resp := webhook.Handle(ctx, Request{}) + + By("checking that it errored the request") + Expect(resp.Allowed).To(BeFalse()) + Expect(resp.Result.Code).To(Equal(int32(http.StatusInternalServerError))) + Expect(resp.Result.Message).To(Equal("panic: fake panic test [recovered]")) }) - It("should pass a setFields that also injects a decoder into sub-dependencies", func() { - By("setting up a webhook and injecting it with a injection func that injects a scheme") - setFields := func(target interface{}) error { - if _, err := inject.SchemeInto(runtime.NewScheme(), target); err != nil { - return err + It("should not recover panic if RecoverPanic is false", func(ctx SpecContext) { + panicHandler := func() *Webhook { + handler := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + panic("fake panic test") + }, } - return nil - } - handler := &handlerWithSubDependencies{ - Handler: HandlerFunc(func(ctx context.Context, req Request) Response { - return Response{} - }), - dep: &subDep{}, - } - webhook := &Webhook{ - Handler: handler, + webhook := &Webhook{ + Handler: handler, + RecoverPanic: ptr.To[bool](false), + } + + return webhook } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) - By("checking that setFields sets the decoder as well") - Expect(handler.dep.decoder).NotTo(BeNil()) + By("setting up a webhook with a panicking handler") + defer func() { + Expect(recover()).ShouldNot(BeNil()) + }() + webhook := panicHandler() + + By("invoking the webhook") + webhook.Handle(ctx, Request{}) }) }) }) -type stringInjector interface { - InjectString(s string) error -} - -type handlerWithSubDependencies struct { - Handler - dep *subDep -} - -func (h *handlerWithSubDependencies) InjectFunc(f inject.Func) error { - return f(h.dep) -} +var _ = It("Should be able to write/read admission.Request to/from context", func(specContext SpecContext) { + testRequest := Request{ + admissionv1.AdmissionRequest{ + UID: "test-uid", + }, + } -type subDep struct { - decoder *Decoder -} + ctx := NewContextWithRequest(specContext, testRequest) -func (d *subDep) InjectDecoder(dec *Decoder) error { - d.decoder = dec - return nil -} + gotRequest, err := RequestFromContext(ctx) + Expect(err).To(Not(HaveOccurred())) + Expect(gotRequest).To(Equal(testRequest)) +}) diff --git a/pkg/webhook/alias.go b/pkg/webhook/alias.go index 293137db49..2882e7bab3 100644 --- a/pkg/webhook/alias.go +++ b/pkg/webhook/alias.go @@ -23,12 +23,6 @@ import ( // define some aliases for common bits of the webhook functionality -// Defaulter defines functions for setting defaults on resources. -type Defaulter = admission.Defaulter - -// Validator defines functions for validating an operation. -type Validator = admission.Validator - // CustomDefaulter defines functions for setting defaults on resources. type CustomDefaulter = admission.CustomDefaulter diff --git a/pkg/webhook/authentication/authentication_suite_test.go b/pkg/webhook/authentication/authentication_suite_test.go index b993d1ef80..29f7b3e17e 100644 --- a/pkg/webhook/authentication/authentication_suite_test.go +++ b/pkg/webhook/authentication/authentication_suite_test.go @@ -19,20 +19,18 @@ package authentication import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestAuthenticationWebhook(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Authentication Webhook Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Authentication Webhook Suite") } var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) -}, 60) +}) diff --git a/pkg/webhook/authentication/doc.go b/pkg/webhook/authentication/doc.go index a1b45c1aef..d2b85f378c 100644 --- a/pkg/webhook/authentication/doc.go +++ b/pkg/webhook/authentication/doc.go @@ -21,9 +21,3 @@ methods to implement authentication webhook handlers. See examples/tokenreview/ for an example of authentication webhooks. */ package authentication - -import ( - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" -) - -var log = logf.RuntimeLog.WithName("authentication") diff --git a/pkg/webhook/authentication/http.go b/pkg/webhook/authentication/http.go index 19f2f9e51c..abf95e5421 100644 --- a/pkg/webhook/authentication/http.go +++ b/pkg/webhook/authentication/http.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" authenticationv1 "k8s.io/api/authentication/v1" @@ -35,6 +34,13 @@ import ( var authenticationScheme = runtime.NewScheme() var authenticationCodecs = serializer.NewCodecFactory(authenticationScheme) +// The TokenReview resource mostly contains a bearer token which +// at most should have a few KB's of size, so we picked 1 MB to +// have plenty of buffer. +// If your use case requires larger max request sizes, please +// open an issue (https://github.com/kubernetes-sigs/controller-runtime/issues/new). +const maxRequestSize = int64(1 * 1024 * 1024) + func init() { utilruntime.Must(authenticationv1.AddToScheme(authenticationScheme)) utilruntime.Must(authenticationv1beta1.AddToScheme(authenticationScheme)) @@ -43,36 +49,38 @@ func init() { var _ http.Handler = &Webhook{} func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var body []byte - var err error ctx := r.Context() if wh.WithContextFunc != nil { ctx = wh.WithContextFunc(ctx, r) } - var reviewResponse Response - if r.Body == nil { - err = errors.New("request body is empty") - wh.log.Error(err, "bad request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + if r.Body == nil || r.Body == http.NoBody { + err := errors.New("request body is empty") + wh.getLogger(nil).Error(err, "bad request") + wh.writeResponse(w, Errored(err)) return } defer r.Body.Close() - if body, err = ioutil.ReadAll(r.Body); err != nil { - wh.log.Error(err, "unable to read the body from the incoming request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + limitedReader := &io.LimitedReader{R: r.Body, N: maxRequestSize} + body, err := io.ReadAll(limitedReader) + if err != nil { + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request") + wh.writeResponse(w, Errored(err)) + return + } + if limitedReader.N <= 0 { + err := fmt.Errorf("request entity is too large; limit is %d bytes", maxRequestSize) + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request; limit reached") + wh.writeResponse(w, Errored(err)) return } // verify the content type is accurate if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { - err = fmt.Errorf("contentType=%s, expected application/json", contentType) - wh.log.Error(err, "unable to process a request with an unknown content type", "content type", contentType) - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + err := fmt.Errorf("contentType=%s, expected application/json", contentType) + wh.getLogger(nil).Error(err, "unable to process a request with unknown content type") + wh.writeResponse(w, Errored(err)) return } @@ -90,23 +98,20 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { ar.SetGroupVersionKind(authenticationv1.SchemeGroupVersion.WithKind("TokenReview")) _, actualTokRevGVK, err := authenticationCodecs.UniversalDeserializer().Decode(body, nil, &ar) if err != nil { - wh.log.Error(err, "unable to decode the request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + wh.getLogger(nil).Error(err, "unable to decode the request") + wh.writeResponse(w, Errored(err)) return } - wh.log.V(1).Info("received request", "UID", req.UID, "kind", req.Kind) + wh.getLogger(&req).V(5).Info("received request") if req.Spec.Token == "" { - err = errors.New("token is empty") - wh.log.Error(err, "bad request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + err := errors.New("token is empty") + wh.getLogger(&req).Error(err, "bad request") + wh.writeResponse(w, Errored(err)) return } - reviewResponse = wh.Handle(ctx, req) - wh.writeResponseTyped(w, reviewResponse, actualTokRevGVK) + wh.writeResponseTyped(w, wh.Handle(ctx, req), actualTokRevGVK) } // writeResponse writes response to w generically, i.e. without encoding GVK information. @@ -132,13 +137,11 @@ func (wh *Webhook) writeResponseTyped(w io.Writer, response Response, tokRevGVK // writeTokenResponse writes ar to w. func (wh *Webhook) writeTokenResponse(w io.Writer, ar authenticationv1.TokenReview) { if err := json.NewEncoder(w).Encode(ar); err != nil { - wh.log.Error(err, "unable to encode the response") + wh.getLogger(nil).Error(err, "unable to encode the response") wh.writeResponse(w, Errored(err)) } res := ar - if log := wh.log; log.V(1).Enabled() { - log.V(1).Info("wrote response", "UID", res.UID, "authenticated", res.Status.Authenticated) - } + wh.getLogger(nil).V(5).Info("wrote response", "requestID", res.UID, "authenticated", res.Status.Authenticated) } // unversionedTokenReview is used to decode both v1 and v1beta1 TokenReview types. diff --git a/pkg/webhook/authentication/http_test.go b/pkg/webhook/authentication/http_test.go index 882a77e427..e51b2af7e6 100644 --- a/pkg/webhook/authentication/http_test.go +++ b/pkg/webhook/authentication/http_test.go @@ -19,17 +19,15 @@ package authentication import ( "bytes" "context" + "crypto/rand" "fmt" "io" "net/http" "net/http/httptest" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("Authentication Webhooks", func() { @@ -47,17 +45,15 @@ var _ = Describe("Authentication Webhooks", func() { respRecorder = &httptest.ResponseRecorder{ Body: bytes.NewBuffer(nil), } - _, err := inject.LoggerInto(log.WithName("test-webhook"), webhook) - Expect(err).NotTo(HaveOccurred()) }) It("should return bad-request when given an empty body", func() { req := &http.Request{Body: nil} - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"request body is empty"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"request body is empty"}} ` webhook.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(expected)) + Expect(respRecorder.Body.String()).To(BeComparableTo(expected)) }) It("should return bad-request when given the wrong content-type", func() { @@ -67,7 +63,7 @@ var _ = Describe("Authentication Webhooks", func() { Body: nopCloser{Reader: bytes.NewBuffer(nil)}, } - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"contentType=application/foo, expected application/json"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"contentType=application/foo, expected application/json"}} ` webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) @@ -80,7 +76,7 @@ var _ = Describe("Authentication Webhooks", func() { Body: nopCloser{Reader: bytes.NewBufferString("{")}, } - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"couldn't get version/kind; json parse error: unexpected end of JSON input"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"couldn't get version/kind; json parse error: unexpected end of JSON input"}} ` webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) @@ -93,7 +89,33 @@ var _ = Describe("Authentication Webhooks", func() { Body: nopCloser{Reader: bytes.NewBufferString(`{"spec":{"token":""}}`)}, } - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"token is empty"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"token is empty"}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + + It("should error when given a NoBody", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: http.NoBody, + } + + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"request body is empty"}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + + It("should error when given an infinite body", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: nopCloser{Reader: rand.Reader}, + } + + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"request entity is too large; limit is 1048576 bytes"}} ` webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) @@ -107,10 +129,9 @@ var _ = Describe("Authentication Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{}}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{}}} `, gvkJSONv1) webhook.ServeHTTP(respRecorder, req) @@ -125,16 +146,15 @@ var _ = Describe("Authentication Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{}}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{}}} `, gvkJSONv1) webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should present the Context from the HTTP request, if any", func() { + It("should present the Context from the HTTP request, if any", func(specContext SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Method: http.MethodPost, @@ -150,19 +170,18 @@ var _ = Describe("Authentication Webhooks", func() { return Authenticated(ctx.Value(key).(string), authenticationv1.UserInfo{}) }, }, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} `, gvkJSONv1, value) - ctx, cancel := context.WithCancel(context.WithValue(context.Background(), key, value)) + ctx, cancel := context.WithCancel(context.WithValue(specContext, key, value)) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should mutate the Context from the HTTP request, if func supplied", func() { + It("should mutate the Context from the HTTP request, if func supplied", func(specContext SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Method: http.MethodPost, @@ -179,13 +198,12 @@ var _ = Describe("Authentication Webhooks", func() { WithContextFunc: func(ctx context.Context, r *http.Request) context.Context { return context.WithValue(ctx, key, r.Header["Content-Type"][0]) }, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} `, gvkJSONv1, "application/json") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specContext) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) @@ -200,14 +218,8 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } type fakeHandler struct { - invoked bool - fn func(context.Context, Request) Response - injectedString string -} - -func (h *fakeHandler) InjectString(s string) error { - h.injectedString = s - return nil + invoked bool + fn func(context.Context, Request) Response } func (h *fakeHandler) Handle(ctx context.Context, req Request) Response { diff --git a/pkg/webhook/authentication/response_test.go b/pkg/webhook/authentication/response_test.go index 22c1ee3ba7..6eeef87c11 100644 --- a/pkg/webhook/authentication/response_test.go +++ b/pkg/webhook/authentication/response_test.go @@ -19,7 +19,7 @@ package authentication import ( "errors" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" diff --git a/pkg/webhook/authentication/webhook.go b/pkg/webhook/authentication/webhook.go index b1229e422e..5a0cd4cd25 100644 --- a/pkg/webhook/authentication/webhook.go +++ b/pkg/webhook/authentication/webhook.go @@ -20,11 +20,13 @@ import ( "context" "errors" "net/http" + "sync" "github.com/go-logr/logr" authenticationv1 "k8s.io/api/authentication/v1" + "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -85,45 +87,40 @@ type Webhook struct { // headers thus allowing you to read them from within the handler WithContextFunc func(context.Context, *http.Request) context.Context - log logr.Logger -} - -// InjectLogger gets a handle to a logging instance, hopefully with more info about this particular webhook. -func (wh *Webhook) InjectLogger(l logr.Logger) error { - wh.log = l - return nil + setupLogOnce sync.Once + log logr.Logger } // Handle processes TokenReview. func (wh *Webhook) Handle(ctx context.Context, req Request) Response { resp := wh.Handler.Handle(ctx, req) if err := resp.Complete(req); err != nil { - wh.log.Error(err, "unable to encode response") + wh.getLogger(&req).Error(err, "unable to encode response") return Errored(errUnableToEncodeResponse) } return resp } -// InjectFunc injects the field setter into the webhook. -func (wh *Webhook) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - - var setFields inject.Func - setFields = func(target interface{}) error { - if err := f(target); err != nil { - return err +// getLogger constructs a logger from the injected log and LogConstructor. +func (wh *Webhook) getLogger(req *Request) logr.Logger { + wh.setupLogOnce.Do(func() { + if wh.log.GetSink() == nil { + wh.log = logf.Log.WithName("authentication") } + }) - if _, err := inject.InjectorInto(setFields, target); err != nil { - return err - } + return logConstructor(wh.log, req) +} - return nil +// logConstructor adds some commonly interesting fields to the given logger. +func logConstructor(base logr.Logger, req *Request) logr.Logger { + if req != nil { + return base.WithValues("object", klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + "user", req.Status.User.Username, + "requestID", req.UID, + ) } - - return setFields(wh.Handler) + return base } diff --git a/pkg/webhook/authentication/webhook_test.go b/pkg/webhook/authentication/webhook_test.go index 55849ece32..22c4e284cd 100644 --- a/pkg/webhook/authentication/webhook_test.go +++ b/pkg/webhook/authentication/webhook_test.go @@ -19,15 +19,12 @@ package authentication import ( "context" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" machinerytypes "k8s.io/apimachinery/pkg/types" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("Authentication Webhooks", func() { @@ -45,46 +42,45 @@ var _ = Describe("Authentication Webhooks", func() { } webhook := &Webhook{ Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), } return webhook } - It("should invoke the handler to get a response", func() { + It("should invoke the handler to get a response", func(ctx SpecContext) { By("setting up a webhook with an allow handler") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that it allowed the request") Expect(resp.Status.Authenticated).To(BeTrue()) }) - It("should ensure that the response's UID is set to the request's UID", func() { + It("should ensure that the response's UID is set to the request's UID", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{TokenReview: authenticationv1.TokenReview{ObjectMeta: metav1.ObjectMeta{UID: "foobar"}}}) + resp := webhook.Handle(ctx, Request{TokenReview: authenticationv1.TokenReview{ObjectMeta: metav1.ObjectMeta{UID: "foobar"}}}) By("checking that the response share's the request's UID") Expect(resp.UID).To(Equal(machinerytypes.UID("foobar"))) }) - It("should populate the status on a response if one is not provided", func() { + It("should populate the status on a response if one is not provided", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the response share's the request's UID") Expect(resp.Status).To(Equal(authenticationv1.TokenReviewStatus{Authenticated: true})) }) - It("shouldn't overwrite the status on a response", func() { + It("shouldn't overwrite the status on a response", func(ctx SpecContext) { By("setting up a webhook that sets a status") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { @@ -97,44 +93,14 @@ var _ = Describe("Authentication Webhooks", func() { }, } }), - log: logf.RuntimeLog.WithName("webhook"), } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the message is intact") Expect(resp.Status).NotTo(BeNil()) Expect(resp.Status.Authenticated).To(BeTrue()) Expect(resp.Status.Error).To(Equal("Ground Control to Major Tom")) }) - - Describe("dependency injection", func() { - It("should set dependencies passed in on the handler", func() { - By("setting up a webhook and injecting it with a injection func that injects a string") - setFields := func(target interface{}) error { - inj, ok := target.(stringInjector) - if !ok { - return nil - } - - return inj.InjectString("something") - } - handler := &fakeHandler{} - webhook := &Webhook{ - Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), - } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) - - By("checking that the string was injected") - Expect(handler.injectedString).To(Equal("something")) - }) - - }) }) - -type stringInjector interface { - InjectString(s string) error -} diff --git a/pkg/webhook/conversion/conversion.go b/pkg/webhook/conversion/conversion.go index a5b7a282ce..a26fa348bb 100644 --- a/pkg/webhook/conversion/conversion.go +++ b/pkg/webhook/conversion/conversion.go @@ -22,7 +22,9 @@ See pkg/conversion for interface definitions required to ensure an API Type is c package conversion import ( + "context" "encoding/json" + "errors" "fmt" "net/http" @@ -31,36 +33,32 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "sigs.k8s.io/controller-runtime/pkg/conversion" logf "sigs.k8s.io/controller-runtime/pkg/log" + conversionmetrics "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/metrics" ) var ( log = logf.Log.WithName("conversion-webhook") ) -// Webhook implements a CRD conversion webhook HTTP handler. -type Webhook struct { - scheme *runtime.Scheme - decoder *Decoder +func NewWebhookHandler(scheme *runtime.Scheme) http.Handler { + return &webhook{scheme: scheme, decoder: NewDecoder(scheme)} } -// InjectScheme injects a scheme into the webhook, in order to construct a Decoder. -func (wh *Webhook) InjectScheme(s *runtime.Scheme) error { - var err error - wh.scheme = s - wh.decoder, err = NewDecoder(s) - if err != nil { - return err - } - - return nil +// webhook implements a CRD conversion webhook HTTP handler. +type webhook struct { + scheme *runtime.Scheme + decoder *Decoder } // ensure Webhook implements http.Handler -var _ http.Handler = &Webhook{} +var _ http.Handler = &webhook{} + +func (wh *webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() -func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { convertReview := &apix.ConversionReview{} err := json.NewDecoder(r.Body).Decode(convertReview) if err != nil { @@ -69,9 +67,15 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if convertReview.Request == nil { + log.Error(nil, "conversion request is nil") + w.WriteHeader(http.StatusBadRequest) + return + } + // TODO(droot): may be move the conversion logic to a separate module to // decouple it from the http layer ? - resp, err := wh.handleConvertRequest(convertReview.Request) + resp, err := wh.handleConvertRequest(ctx, convertReview.Request) if err != nil { log.Error(err, "failed to convert", "request", convertReview.Request.UID) convertReview.Response = errored(err) @@ -89,7 +93,18 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // handles a version conversion request. -func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) { +func (wh *webhook) handleConvertRequest(ctx context.Context, req *apix.ConversionRequest) (_ *apix.ConversionResponse, retErr error) { + defer func() { + if r := recover(); r != nil { + conversionmetrics.WebhookPanics.WithLabelValues().Inc() + + for _, fn := range utilruntime.PanicHandlers { + fn(ctx, r) + } + retErr = errors.New("internal error occurred during conversion") + return + } + }() if req == nil { return nil, fmt.Errorf("conversion request is nil") } @@ -122,7 +137,7 @@ func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.Conv // convertObject will convert given a src object to dst object. // Note(droot): couldn't find a way to reduce the cyclomatic complexity under 10 // without compromising readability, so disabling gocyclo linter -func (wh *Webhook) convertObject(src, dst runtime.Object) error { +func (wh *webhook) convertObject(src, dst runtime.Object) error { srcGVK := src.GetObjectKind().GroupVersionKind() dstGVK := dst.GetObjectKind().GroupVersionKind() @@ -149,7 +164,7 @@ func (wh *Webhook) convertObject(src, dst runtime.Object) error { } } -func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error { +func (wh *webhook) convertViaHub(src, dst conversion.Convertible) error { hub, err := wh.getHub(src) if err != nil { return err @@ -173,7 +188,7 @@ func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error { } // getHub returns an instance of the Hub for passed-in object's group/kind. -func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) { +func (wh *webhook) getHub(obj runtime.Object) (conversion.Hub, error) { gvks, err := objectGVKs(wh.scheme, obj) if err != nil { return nil, err @@ -201,7 +216,7 @@ func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) { } // allocateDstObject returns an instance for a given GVK. -func (wh *Webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) { +func (wh *webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) { gvk := schema.FromAPIVersionAndKind(apiVersion, kind) obj, err := wh.scheme.New(gvk) diff --git a/pkg/webhook/conversion/conversion_suite_test.go b/pkg/webhook/conversion/conversion_suite_test.go index bb3798747c..7ca3c48ba2 100644 --- a/pkg/webhook/conversion/conversion_suite_test.go +++ b/pkg/webhook/conversion/conversion_suite_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -18,18 +18,16 @@ package conversion import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestConversionWebhook(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "CRD conversion Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "CRD conversion Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/webhook/conversion/conversion_test.go b/pkg/webhook/conversion/conversion_test.go index 26a6587654..489689bccb 100644 --- a/pkg/webhook/conversion/conversion_test.go +++ b/pkg/webhook/conversion/conversion_test.go @@ -14,16 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package conversion +package conversion_test import ( "bytes" "encoding/json" - "io/ioutil" + "io" "net/http" "net/http/httptest" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1beta1 "k8s.io/api/apps/v1beta1" @@ -33,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" kscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/webhook/conversion" jobsv1 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v1" jobsv2 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v2" jobsv3 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v3" @@ -41,9 +42,9 @@ import ( var _ = Describe("Conversion Webhook", func() { var respRecorder *httptest.ResponseRecorder - var decoder *Decoder + var decoder *conversion.Decoder var scheme *runtime.Scheme - webhook := Webhook{} + var wh http.Handler BeforeEach(func() { respRecorder = &httptest.ResponseRecorder{ @@ -55,12 +56,9 @@ var _ = Describe("Conversion Webhook", func() { Expect(jobsv1.AddToScheme(scheme)).To(Succeed()) Expect(jobsv2.AddToScheme(scheme)).To(Succeed()) Expect(jobsv3.AddToScheme(scheme)).To(Succeed()) - Expect(webhook.InjectScheme(scheme)).To(Succeed()) - - var err error - decoder, err = NewDecoder(scheme) - Expect(err).NotTo(HaveOccurred()) + decoder = conversion.NewDecoder(scheme) + wh = conversion.NewWebhookHandler(scheme) }) doRequest := func(convReq *apix.ConversionReview) *apix.ConversionReview { @@ -70,9 +68,9 @@ var _ = Describe("Conversion Webhook", func() { convReview := &apix.ConversionReview{} req := &http.Request{ - Body: ioutil.NopCloser(bytes.NewReader(payload.Bytes())), + Body: io.NopCloser(bytes.NewReader(payload.Bytes())), } - webhook.ServeHTTP(respRecorder, req) + wh.ServeHTTP(respRecorder, req) Expect(json.NewDecoder(respRecorder.Result().Body).Decode(convReview)).To(Succeed()) return convReview } @@ -298,6 +296,29 @@ var _ = Describe("Conversion Webhook", func() { Expect(convReview.Response.ConvertedObjects).To(BeEmpty()) }) + It("should return error on panic in conversion", func() { + + v1Obj := makeV1Obj() + v1Obj.Spec.PanicInConversion = true + + convReq := &apix.ConversionReview{ + TypeMeta: metav1.TypeMeta{}, + Request: &apix.ConversionRequest{ + DesiredAPIVersion: "jobs.testprojects.kb.io/v3", + Objects: []runtime.RawExtension{ + { + Object: v1Obj, + }, + }, + }, + } + + convReview := doRequest(convReq) + + Expect(convReview.Response.ConvertedObjects).To(HaveLen(0)) + Expect(convReview.Response.Result.Status).To(Equal(metav1.StatusFailure)) + Expect(convReview.Response.Result.Message).To(Equal("internal error occurred during conversion")) + }) }) var _ = Describe("IsConvertible", func() { @@ -316,7 +337,7 @@ var _ = Describe("IsConvertible", func() { It("should not error for uninitialized types", func() { obj := &jobsv2.ExternalJob{} - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).To(BeTrue()) }) @@ -329,7 +350,7 @@ var _ = Describe("IsConvertible", func() { }, } - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).To(BeTrue()) }) @@ -342,7 +363,7 @@ var _ = Describe("IsConvertible", func() { }, } - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).To(BeTrue()) }) @@ -355,7 +376,7 @@ var _ = Describe("IsConvertible", func() { }, } - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).ToNot(BeTrue()) }) diff --git a/pkg/webhook/conversion/decoder.go b/pkg/webhook/conversion/decoder.go index 6a9e9c2365..b6bb8bd938 100644 --- a/pkg/webhook/conversion/decoder.go +++ b/pkg/webhook/conversion/decoder.go @@ -30,8 +30,11 @@ type Decoder struct { } // NewDecoder creates a Decoder given the runtime.Scheme -func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) { - return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil +func NewDecoder(scheme *runtime.Scheme) *Decoder { + if scheme == nil { + panic("scheme should never be nil") + } + return &Decoder{codecs: serializer.NewCodecFactory(scheme)} } // Decode decodes the inlined object. diff --git a/pkg/webhook/conversion/metrics/metrics.go b/pkg/webhook/conversion/metrics/metrics.go new file mode 100644 index 0000000000..c825f17f0b --- /dev/null +++ b/pkg/webhook/conversion/metrics/metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // WebhookPanics is a prometheus counter metrics which holds the total + // number of panics from conversion webhooks. + WebhookPanics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "controller_runtime_conversion_webhook_panics_total", + Help: "Total number of conversion webhook panics", + }, []string{}) +) + +func init() { + metrics.Registry.MustRegister( + WebhookPanics, + ) + // Init metric. + WebhookPanics.WithLabelValues().Add(0) +} diff --git a/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go b/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go index bf99e2a204..c6065e1fb4 100644 --- a/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go +++ b/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go @@ -17,6 +17,7 @@ package v1 import ( "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -31,6 +32,9 @@ type ExternalJobSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file RunAt string `json:"runAt"` + + // PanicInConversion triggers a panic during conversion when set to true. + PanicInConversion bool `json:"panicInConversion"` } // ExternalJobStatus defines the observed state of ExternalJob @@ -66,6 +70,9 @@ func init() { // ConvertTo implements conversion logic to convert to Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := dst.(type) { case *v2.ExternalJob: jobv2 := dst.(*v2.ExternalJob) @@ -80,6 +87,9 @@ func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { // ConvertFrom implements conversion logic to convert from Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertFrom(src conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := src.(type) { case *v2.ExternalJob: jobv2 := src.(*v2.ExternalJob) diff --git a/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go b/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go index 7208ba8c69..af7396abf1 100644 --- a/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go +++ b/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -// +build !ignore_autogenerated +//go:build !ignore_autogenerated /* @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// autogenerated by controller-gen object, do not modify manually +// Code generated by controller-gen. DO NOT EDIT. package v1 @@ -54,7 +54,7 @@ func (in *ExternalJob) DeepCopyObject() runtime.Object { func (in *ExternalJobList) DeepCopyInto(out *ExternalJobList) { *out = *in out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ExternalJob, len(*in)) diff --git a/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go b/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go index de5a03a212..1f87e8a017 100644 --- a/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go +++ b/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go @@ -27,6 +27,9 @@ type ExternalJobSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file ScheduleAt string `json:"scheduleAt"` + + // PanicInConversion triggers a panic during conversion when set to true. + PanicInConversion bool `json:"panicInConversion"` } // ExternalJobStatus defines the observed state of ExternalJob diff --git a/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go b/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go index 53c9f758b1..d5efd6150e 100644 --- a/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go +++ b/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -// +build !ignore_autogenerated +//go:build !ignore_autogenerated /* @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// autogenerated by controller-gen object, do not modify manually +// Code generated by controller-gen. DO NOT EDIT. package v2 @@ -54,7 +54,7 @@ func (in *ExternalJob) DeepCopyObject() runtime.Object { func (in *ExternalJobList) DeepCopyInto(out *ExternalJobList) { *out = *in out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ExternalJob, len(*in)) diff --git a/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go b/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go index 15c438f68a..85a166b7cf 100644 --- a/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go +++ b/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go @@ -17,6 +17,7 @@ package v3 import ( "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -31,6 +32,9 @@ type ExternalJobSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file DeferredAt string `json:"deferredAt"` + + // PanicInConversion triggers a panic during conversion when set to true. + PanicInConversion bool `json:"panicInConversion"` } // ExternalJobStatus defines the observed state of ExternalJob @@ -66,6 +70,9 @@ func init() { // ConvertTo implements conversion logic to convert to Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := dst.(type) { case *v2.ExternalJob: jobv2 := dst.(*v2.ExternalJob) @@ -80,6 +87,9 @@ func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { // ConvertFrom implements conversion logic to convert from Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertFrom(src conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := src.(type) { case *v2.ExternalJob: jobv2 := src.(*v2.ExternalJob) diff --git a/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go b/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go index a90942b427..d12b6910dc 100644 --- a/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go +++ b/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -// +build !ignore_autogenerated +//go:build !ignore_autogenerated /* @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// autogenerated by controller-gen object, do not modify manually +// Code generated by controller-gen. DO NOT EDIT. package v3 @@ -54,7 +54,7 @@ func (in *ExternalJob) DeepCopyObject() runtime.Object { func (in *ExternalJobList) DeepCopyInto(out *ExternalJobList) { *out = *in out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ExternalJob, len(*in)) diff --git a/pkg/webhook/conversion/testdata/main.go b/pkg/webhook/conversion/testdata/main.go index a3922da009..ea6b8275a8 100644 --- a/pkg/webhook/conversion/testdata/main.go +++ b/pkg/webhook/conversion/testdata/main.go @@ -25,6 +25,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" jobsv1 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v1" jobsv2 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v2" jobsv3 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v3" @@ -55,9 +56,9 @@ func main() { ctrl.SetLogger(zap.Logger(true)) mgr, err := ctrl.NewManager(context.Background(), ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: metricsAddr}, + LeaderElection: enableLeaderElection, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/pkg/webhook/example_test.go b/pkg/webhook/example_test.go index e1f2bbee6c..7c4f718f4c 100644 --- a/pkg/webhook/example_test.go +++ b/pkg/webhook/example_test.go @@ -20,7 +20,6 @@ import ( "context" "net/http" - "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -62,9 +61,9 @@ func Example() { } // Create a webhook server. - hookServer := &Server{ + hookServer := NewServer(Options{ Port: 8443, - } + }) if err := mgr.Add(hookServer); err != nil { panic(err) } @@ -87,18 +86,18 @@ func Example() { // Note that this assumes and requires a valid TLS // cert and key at the default locations // tls.crt and tls.key. -func ExampleServer_StartStandalone() { +func ExampleServer_Start() { // Create a webhook server - hookServer := &Server{ + hookServer := NewServer(Options{ Port: 8443, - } + }) // Register the webhooks in the server. hookServer.Register("/mutating", mutatingHook) hookServer.Register("/validating", validatingHook) // Start the server without a manger - err := hookServer.StartStandalone(signals.SetupSignalHandler(), scheme.Scheme) + err := hookServer.Start(signals.SetupSignalHandler()) if err != nil { // handle error panic(err) @@ -118,7 +117,6 @@ func ExampleStandaloneWebhook() { // Create the standalone HTTP handlers from our webhooks mutatingHookHandler, err := admission.StandaloneWebhook(mutatingHook, admission.StandaloneOptions{ - Scheme: scheme.Scheme, // Logger let's you optionally pass // a custom logger (defaults to log.Log global Logger) Logger: logf.RuntimeLog.WithName("mutating-webhook"), @@ -134,7 +132,6 @@ func ExampleStandaloneWebhook() { } validatingHookHandler, err := admission.StandaloneWebhook(validatingHook, admission.StandaloneOptions{ - Scheme: scheme.Scheme, Logger: logf.RuntimeLog.WithName("validating-webhook"), MetricsPath: "/validating", }) diff --git a/pkg/webhook/internal/metrics/metrics.go b/pkg/webhook/internal/metrics/metrics.go index 557004908b..f1e6ce68f5 100644 --- a/pkg/webhook/internal/metrics/metrics.go +++ b/pkg/webhook/internal/metrics/metrics.go @@ -18,6 +18,7 @@ package metrics import ( "net/http" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -30,8 +31,11 @@ var ( // of processing admission requests. RequestLatency = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "controller_runtime_webhook_latency_seconds", - Help: "Histogram of the latency of processing admission requests", + Name: "controller_runtime_webhook_latency_seconds", + Help: "Histogram of the latency of processing admission requests", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"webhook"}, ) diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index d2338d0b77..4d8ae9ec7a 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -21,7 +21,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" "net" "net/http" "os" @@ -30,11 +29,9 @@ import ( "sync" "time" - "k8s.io/apimachinery/pkg/runtime" - kscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" ) @@ -49,7 +46,29 @@ var DefaultPort = 9443 // at the default locations (tls.crt and tls.key). If you do not // want to configure TLS (i.e for testing purposes) run an // admission.StandaloneWebhook in your own server. -type Server struct { +type Server interface { + // NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates + // the webhook server doesn't need leader election. + NeedLeaderElection() bool + + // Register marks the given webhook as being served at the given path. + // It panics if two hooks are registered on the same path. + Register(path string, hook http.Handler) + + // Start runs the server. + // It will install the webhook related resources depend on the server configuration. + Start(ctx context.Context) error + + // StartedChecker returns an healthz.Checker which is healthy after the + // server has been started. + StartedChecker() healthz.Checker + + // WebhookMux returns the servers WebhookMux + WebhookMux() *http.ServeMux +} + +// Options are all the available options for a webhook.Server +type Options struct { // Host is the address that the server will listen on. // Defaults to "" - all addresses. Host string @@ -58,33 +77,45 @@ type Server struct { // It will be defaulted to 9443 if unspecified. Port int - // CertDir is the directory that contains the server key and certificate. The - // server key and certificate. + // CertDir is the directory that contains the server key and certificate. Defaults to + // /k8s-webhook-server/serving-certs. CertDir string // CertName is the server certificate name. Defaults to tls.crt. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. CertName string // KeyName is the server key name. Defaults to tls.key. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. KeyName string // ClientCAName is the CA certificate name which server used to verify remote(client)'s certificate. // Defaults to "", which means server does not verify client's certificate. ClientCAName string - // TLSVersion is the minimum version of TLS supported. Accepts - // "", "1.0", "1.1", "1.2" and "1.3" only ("" is equivalent to "1.0" for backwards compatibility) - TLSMinVersion string + // TLSOpts is used to allow configuring the TLS config used for the server. + // This also allows providing a certificate via GetCertificate. + TLSOpts []func(*tls.Config) // WebhookMux is the multiplexer that handles different webhooks. WebhookMux *http.ServeMux +} - // webhooks keep track of all registered webhooks for dependency injection, - // and to provide better panic messages on duplicate webhook registration. - webhooks map[string]http.Handler +// NewServer constructs a new webhook.Server from the provided options. +func NewServer(o Options) Server { + return &DefaultServer{ + Options: o, + } +} - // setFields allows injecting dependencies from an external source - setFields inject.Func +// DefaultServer is the default implementation used for Server. +type DefaultServer struct { + Options Options + + // webhooks keep track of all registered webhooks + webhooks map[string]http.Handler // defaultingOnce ensures that the default fields are only ever set once. defaultingOnce sync.Once @@ -95,41 +126,49 @@ type Server struct { // mu protects access to the webhook map & setFields for Start, Register, etc mu sync.Mutex + + webhookMux *http.ServeMux } // setDefaults does defaulting for the Server. -func (s *Server) setDefaults() { - s.webhooks = map[string]http.Handler{} - if s.WebhookMux == nil { - s.WebhookMux = http.NewServeMux() +func (o *Options) setDefaults() { + if o.WebhookMux == nil { + o.WebhookMux = http.NewServeMux() } - if s.Port <= 0 { - s.Port = DefaultPort + if o.Port <= 0 { + o.Port = DefaultPort } - if len(s.CertDir) == 0 { - s.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs") + if len(o.CertDir) == 0 { + o.CertDir = filepath.Join(os.TempDir(), "k8s-webhook-server", "serving-certs") } - if len(s.CertName) == 0 { - s.CertName = "tls.crt" + if len(o.CertName) == 0 { + o.CertName = "tls.crt" } - if len(s.KeyName) == 0 { - s.KeyName = "tls.key" + if len(o.KeyName) == 0 { + o.KeyName = "tls.key" } } +func (s *DefaultServer) setDefaults() { + s.webhooks = map[string]http.Handler{} + s.Options.setDefaults() + + s.webhookMux = s.Options.WebhookMux +} + // NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates // the webhook server doesn't need leader election. -func (*Server) NeedLeaderElection() bool { +func (*DefaultServer) NeedLeaderElection() bool { return false } // Register marks the given webhook as being served at the given path. // It panics if two hooks are registered on the same path. -func (s *Server) Register(path string, hook http.Handler) { +func (s *DefaultServer) Register(path string, hook http.Handler) { s.mu.Lock() defer s.mu.Unlock() @@ -137,112 +176,53 @@ func (s *Server) Register(path string, hook http.Handler) { if _, found := s.webhooks[path]; found { panic(fmt.Errorf("can't register duplicate path: %v", path)) } - // TODO(directxman12): call setfields if we've already started the server s.webhooks[path] = hook - s.WebhookMux.Handle(path, metrics.InstrumentedHook(path, hook)) + s.webhookMux.Handle(path, metrics.InstrumentedHook(path, hook)) regLog := log.WithValues("path", path) - regLog.Info("registering webhook") - - // we've already been "started", inject dependencies here. - // Otherwise, InjectFunc will do this for us later. - if s.setFields != nil { - if err := s.setFields(hook); err != nil { - // TODO(directxman12): swallowing this error isn't great, but we'd have to - // change the signature to fix that - regLog.Error(err, "unable to inject fields into webhook during registration") - } - - baseHookLog := log.WithName("webhooks") - - // NB(directxman12): we don't propagate this further by wrapping setFields because it's - // unclear if this is how we want to deal with log propagation. In this specific instance, - // we want to be able to pass a logger to webhooks because they don't know their own path. - if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", path), hook); err != nil { - regLog.Error(err, "unable to logger into webhook during registration") - } - } -} - -// StartStandalone runs a webhook server without -// a controller manager. -func (s *Server) StartStandalone(ctx context.Context, scheme *runtime.Scheme) error { - // Use the Kubernetes client-go scheme if none is specified - if scheme == nil { - scheme = kscheme.Scheme - } - - if err := s.InjectFunc(func(i interface{}) error { - if _, err := inject.SchemeInto(scheme, i); err != nil { - return err - } - return nil - }); err != nil { - return err - } - - return s.Start(ctx) -} - -// tlsVersion converts from human-readable TLS version (for example "1.1") -// to the values accepted by tls.Config (for example 0x301). -func tlsVersion(version string) (uint16, error) { - switch version { - // default is previous behaviour - case "": - return tls.VersionTLS10, nil - case "1.0": - return tls.VersionTLS10, nil - case "1.1": - return tls.VersionTLS11, nil - case "1.2": - return tls.VersionTLS12, nil - case "1.3": - return tls.VersionTLS13, nil - default: - return 0, fmt.Errorf("invalid TLSMinVersion %v: expects 1.0, 1.1, 1.2, 1.3 or empty", version) - } + regLog.Info("Registering webhook") } // Start runs the server. // It will install the webhook related resources depend on the server configuration. -func (s *Server) Start(ctx context.Context) error { +func (s *DefaultServer) Start(ctx context.Context) error { s.defaultingOnce.Do(s.setDefaults) - baseHookLog := log.WithName("webhooks") - baseHookLog.Info("starting webhook server") - - certPath := filepath.Join(s.CertDir, s.CertName) - keyPath := filepath.Join(s.CertDir, s.KeyName) + log.Info("Starting webhook server") - certWatcher, err := certwatcher.New(certPath, keyPath) - if err != nil { - return err + cfg := &tls.Config{ + NextProtos: []string{"h2"}, + } + // fallback TLS config ready, will now mutate if passer wants full control over it + for _, op := range s.Options.TLSOpts { + op(cfg) } - go func() { - if err := certWatcher.Start(ctx); err != nil { - log.Error(err, "certificate watcher error") - } - }() + if cfg.GetCertificate == nil { + certPath := filepath.Join(s.Options.CertDir, s.Options.CertName) + keyPath := filepath.Join(s.Options.CertDir, s.Options.KeyName) - tlsMinVersion, err := tlsVersion(s.TLSMinVersion) - if err != nil { - return err - } + // Create the certificate watcher and + // set the config's GetCertificate on the TLSConfig + certWatcher, err := certwatcher.New(certPath, keyPath) + if err != nil { + return err + } + cfg.GetCertificate = certWatcher.GetCertificate - cfg := &tls.Config{ //nolint:gosec - NextProtos: []string{"h2"}, - GetCertificate: certWatcher.GetCertificate, - MinVersion: tlsMinVersion, + go func() { + if err := certWatcher.Start(ctx); err != nil { + log.Error(err, "certificate watcher error") + } + }() } - // load CA to verify client certificate - if s.ClientCAName != "" { + // Load CA to verify client certificate, if configured. + if s.Options.ClientCAName != "" { certPool := x509.NewCertPool() - clientCABytes, err := ioutil.ReadFile(filepath.Join(s.CertDir, s.ClientCAName)) + clientCABytes, err := os.ReadFile(filepath.Join(s.Options.CertDir, s.Options.ClientCAName)) if err != nil { - return fmt.Errorf("failed to read client CA cert: %v", err) + return fmt.Errorf("failed to read client CA cert: %w", err) } ok := certPool.AppendCertsFromPEM(clientCABytes) @@ -254,24 +234,23 @@ func (s *Server) Start(ctx context.Context) error { cfg.ClientAuth = tls.RequireAndVerifyClientCert } - listener, err := tls.Listen("tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), cfg) + listener, err := tls.Listen("tcp", net.JoinHostPort(s.Options.Host, strconv.Itoa(s.Options.Port)), cfg) if err != nil { return err } - log.Info("serving webhook server", "host", s.Host, "port", s.Port) + log.Info("Serving webhook server", "host", s.Options.Host, "port", s.Options.Port) - srv := &http.Server{ - Handler: s.WebhookMux, - } + srv := httpserver.New(s.webhookMux) idleConnsClosed := make(chan struct{}) go func() { <-ctx.Done() - log.Info("shutting down webhook server") + log.Info("Shutting down webhook server with timeout of 1 minute") - // TODO: use a context with reasonable timeout - if err := srv.Shutdown(context.Background()); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { // Error from closing listeners, or context timeout log.Error(err, "error shutting down the HTTP server") } @@ -291,9 +270,9 @@ func (s *Server) Start(ctx context.Context) error { // StartedChecker returns an healthz.Checker which is healthy after the // server has been started. -func (s *Server) StartedChecker() healthz.Checker { +func (s *DefaultServer) StartedChecker() healthz.Checker { config := &tls.Config{ - InsecureSkipVerify: true, // nolint:gosec // config is used to connect to our own webhook port. + InsecureSkipVerify: true, } return func(req *http.Request) error { s.mu.Lock() @@ -304,36 +283,20 @@ func (s *Server) StartedChecker() healthz.Checker { } d := &net.Dialer{Timeout: 10 * time.Second} - conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), config) + conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Options.Host, strconv.Itoa(s.Options.Port)), config) if err != nil { - return fmt.Errorf("webhook server is not reachable: %v", err) + return fmt.Errorf("webhook server is not reachable: %w", err) } if err := conn.Close(); err != nil { - return fmt.Errorf("webhook server is not reachable: closing connection: %v", err) + return fmt.Errorf("webhook server is not reachable: closing connection: %w", err) } return nil } } -// InjectFunc injects the field setter into the server. -func (s *Server) InjectFunc(f inject.Func) error { - s.setFields = f - - // inject fields here that weren't injected in Register because we didn't have setFields yet. - baseHookLog := log.WithName("webhooks") - for hookPath, webhook := range s.webhooks { - if err := s.setFields(webhook); err != nil { - return err - } - - // NB(directxman12): we don't propagate this further by wrapping setFields because it's - // unclear if this is how we want to deal with log propagation. In this specific instance, - // we want to be able to pass a logger to webhooks because they don't know their own path. - if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", hookPath), webhook); err != nil { - return err - } - } - return nil +// WebhookMux returns the servers WebhookMux +func (s *DefaultServer) WebhookMux() *http.ServeMux { + return s.webhookMux } diff --git a/pkg/webhook/server_test.go b/pkg/webhook/server_test.go index 03323eede8..6542222585 100644 --- a/pkg/webhook/server_test.go +++ b/pkg/webhook/server_test.go @@ -18,32 +18,37 @@ package webhook_test import ( "context" + "crypto/tls" "fmt" - "io/ioutil" + "io" "net" "net/http" + "path" + "reflect" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("Webhook Server", func() { var ( - ctx context.Context - ctxCancel context.CancelFunc - testHostPort string - client *http.Client - server *webhook.Server - servingOpts envtest.WebhookInstallOptions + ctxCancel context.CancelFunc + testHostPort string + client *http.Client + server webhook.Server + servingOpts envtest.WebhookInstallOptions + genericStartServer func(f func(ctx context.Context)) (done <-chan struct{}) ) BeforeEach(func() { - ctx, ctxCancel = context.WithCancel(context.Background()) - // closed in individual tests differently + var ctx context.Context + // Has to be derived from context.Background() as it needs to be + // valid past the BeforeEach + ctx, ctxCancel = context.WithCancel(context.Background()) //nolint:forbidigo servingOpts = envtest.WebhookInstallOptions{} Expect(servingOpts.PrepWithoutInstalling()).To(Succeed()) @@ -59,41 +64,32 @@ var _ = Describe("Webhook Server", func() { Transport: clientTransport, } - server = &webhook.Server{ + server = webhook.NewServer(webhook.Options{ Host: servingOpts.LocalServingHost, Port: servingOpts.LocalServingPort, CertDir: servingOpts.LocalServingCertDir, + }) + + genericStartServer = func(f func(ctx context.Context)) (done <-chan struct{}) { + doneCh := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(doneCh) + f(ctx) + }() + // wait till we can ping the server to start the test + Eventually(func() error { + _, err := client.Get(fmt.Sprintf("https://%s/unservedpath", testHostPort)) + return err + }).Should(Succeed()) + + return doneCh } }) AfterEach(func() { Expect(servingOpts.Cleanup()).To(Succeed()) }) - genericStartServer := func(f func(ctx context.Context)) (done <-chan struct{}) { - doneCh := make(chan struct{}) - go func() { - defer GinkgoRecover() - defer close(doneCh) - f(ctx) - }() - // wait till we can ping the server to start the test - Eventually(func() error { - _, err := client.Get(fmt.Sprintf("https://%s/unservedpath", testHostPort)) - return err - }).Should(Succeed()) - - // this is normally called before Start by the manager - Expect(server.InjectFunc(func(i interface{}) error { - boolInj, canInj := i.(interface{ InjectBool(bool) error }) - if !canInj { - return nil - } - return boolInj.InjectBool(true) - })).To(Succeed()) - - return doneCh - } - startServer := func() (done <-chan struct{}) { return genericStartServer(func(ctx context.Context) { Expect(server.Start(ctx)).To(Succeed()) @@ -136,7 +132,7 @@ var _ = Describe("Webhook Server", func() { resp, err := client.Get(fmt.Sprintf("https://%s/somepath", testHostPort)) Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) }).Should(Equal([]byte("gadzooks!"))) Expect(server.StartedChecker()(nil)).To(Succeed()) @@ -144,17 +140,6 @@ var _ = Describe("Webhook Server", func() { ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) }) - - It("should inject dependencies eventually, given an inject func is eventually provided", func() { - handler := &testHandler{} - server.Register("/somepath", handler) - doneCh := startServer() - - Eventually(func() bool { return handler.injectedField }).Should(BeTrue()) - - ctxCancel() - Eventually(doneCh, "4s").Should(BeClosed()) - }) }) Context("when registering webhooks after starting", func() { @@ -176,33 +161,93 @@ var _ = Describe("Webhook Server", func() { Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() - Expect(ioutil.ReadAll(resp.Body)).To(Equal([]byte("gadzooks!"))) + Expect(io.ReadAll(resp.Body)).To(Equal([]byte("gadzooks!"))) }) + }) - It("should inject dependencies, if an inject func has been provided already", func() { - handler := &testHandler{} - server.Register("/somepath", handler) - Expect(handler.injectedField).To(BeTrue()) + It("should respect passed in TLS configurations", func() { + var finalCfg *tls.Config + tlsCfgFunc := func(cfg *tls.Config) { + cfg.CipherSuites = []uint16{ + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + } + cfg.MinVersion = tls.VersionTLS12 + // save cfg after changes to test against + finalCfg = cfg + } + server = webhook.NewServer(webhook.Options{ + Host: servingOpts.LocalServingHost, + Port: servingOpts.LocalServingPort, + CertDir: servingOpts.LocalServingCertDir, + TLSOpts: []func(*tls.Config){ + tlsCfgFunc, + }, + }) + server.Register("/somepath", &testHandler{}) + doneCh := genericStartServer(func(ctx context.Context) { + Expect(server.Start(ctx)).To(Succeed()) }) + + Eventually(func() ([]byte, error) { + resp, err := client.Get(fmt.Sprintf("https://%s/somepath", testHostPort)) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + return io.ReadAll(resp.Body) + }).Should(Equal([]byte("gadzooks!"))) + Expect(finalCfg.MinVersion).To(Equal(uint16(tls.VersionTLS12))) + Expect(finalCfg.CipherSuites).To(ContainElements( + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + )) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) }) - It("should serve be able to serve in unmanaged mode", func() { - server = &webhook.Server{ + It("should prefer GetCertificate through TLSOpts", func() { + var finalCfg *tls.Config + finalCert, err := tls.LoadX509KeyPair( + path.Join(servingOpts.LocalServingCertDir, "tls.crt"), + path.Join(servingOpts.LocalServingCertDir, "tls.key"), + ) + Expect(err).NotTo(HaveOccurred()) + finalGetCertificate := func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { //nolint:unparam + return &finalCert, nil + } + server = &webhook.DefaultServer{Options: webhook.Options{ Host: servingOpts.LocalServingHost, Port: servingOpts.LocalServingPort, CertDir: servingOpts.LocalServingCertDir, - } + + TLSOpts: []func(*tls.Config){ + func(cfg *tls.Config) { + cfg.GetCertificate = finalGetCertificate + cfg.MinVersion = tls.VersionTLS12 + // save cfg after changes to test against + finalCfg = cfg + }, + }, + }} server.Register("/somepath", &testHandler{}) doneCh := genericStartServer(func(ctx context.Context) { - Expect(server.StartStandalone(ctx, scheme.Scheme)) + Expect(server.Start(ctx)).To(Succeed()) }) Eventually(func() ([]byte, error) { resp, err := client.Get(fmt.Sprintf("https://%s/somepath", testHostPort)) Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) }).Should(Equal([]byte("gadzooks!"))) + Expect(finalCfg.MinVersion).To(Equal(uint16(tls.VersionTLS12))) + // We can't compare the functions directly, but we can compare their pointers + if reflect.ValueOf(finalCfg.GetCertificate).Pointer() != reflect.ValueOf(finalGetCertificate).Pointer() { + Fail("GetCertificate was not set properly, or overwritten") + } + cert, err := finalCfg.GetCertificate(nil) + Expect(err).NotTo(HaveOccurred()) + Expect(cert).To(BeEquivalentTo(&finalCert)) ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) @@ -210,13 +255,8 @@ var _ = Describe("Webhook Server", func() { }) type testHandler struct { - injectedField bool } -func (t *testHandler) InjectBool(val bool) error { - t.injectedField = val - return nil -} func (t *testHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { if _, err := resp.Write([]byte("gadzooks!")); err != nil { panic("unable to write http response!") diff --git a/pkg/webhook/webhook_integration_test.go b/pkg/webhook/webhook_integration_test.go index 0b9754ef40..cbb5b711f7 100644 --- a/pkg/webhook/webhook_integration_test.go +++ b/pkg/webhook/webhook_integration_test.go @@ -19,27 +19,21 @@ package webhook_test import ( "context" "crypto/tls" - "errors" - "net" - "net/http" - "path/filepath" - "strconv" + "strings" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/certwatcher" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission/admissiontest" ) var _ = Describe("Webhook", func() { @@ -79,141 +73,79 @@ var _ = Describe("Webhook", func() { } }) Context("when running a webhook server with a manager", func() { - It("should reject create request for webhook that rejects all requests", func() { + It("should reject create request for webhook that rejects all requests", func(ctx SpecContext) { m, err := manager.New(cfg, manager.Options{ - Port: testenv.WebhookInstallOptions.LocalServingPort, - Host: testenv.WebhookInstallOptions.LocalServingHost, - CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: testenv.WebhookInstallOptions.LocalServingPort, + Host: testenv.WebhookInstallOptions.LocalServingHost, + CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + }), }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() - server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) + server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{d: admission.NewDecoder(testenv.Scheme)}}) - ctx, cancel := context.WithCancel(context.Background()) go func() { - err = server.Start(ctx) + err := server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { - err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + err := c.Create(ctx, obj) + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) - - cancel() }) - It("should reject create request for multi-webhook that rejects all requests", func() { + It("should reject create request for multi-webhook that rejects all requests", func(ctx SpecContext) { m, err := manager.New(cfg, manager.Options{ - Port: testenv.WebhookInstallOptions.LocalServingPort, - Host: testenv.WebhookInstallOptions.LocalServingHost, - CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + Metrics: metricsserver.Options{BindAddress: "0"}, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: testenv.WebhookInstallOptions.LocalServingPort, + Host: testenv.WebhookInstallOptions.LocalServingHost, + CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + }), }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() - server.Register("/failing", &webhook.Admission{Handler: admission.MultiValidatingHandler(&rejectingValidator{})}) + server.Register("/failing", &webhook.Admission{Handler: admission.MultiValidatingHandler(&rejectingValidator{d: admission.NewDecoder(testenv.Scheme)})}) - ctx, cancel := context.WithCancel(context.Background()) go func() { + defer GinkgoRecover() err = server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { - err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + err = c.Create(ctx, obj) + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) - - cancel() }) }) Context("when running a webhook server without a manager", func() { - It("should reject create request for webhook that rejects all requests", func() { - server := webhook.Server{ + It("should reject create request for webhook that rejects all requests", func(ctx SpecContext) { + server := webhook.NewServer(webhook.Options{ Port: testenv.WebhookInstallOptions.LocalServingPort, Host: testenv.WebhookInstallOptions.LocalServingHost, CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, - } - server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) + }) + server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{d: admission.NewDecoder(testenv.Scheme)}}) - ctx, cancel := context.WithCancel(context.Background()) go func() { - err := server.StartStandalone(ctx, scheme.Scheme) + err := server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { - err := c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") - }, 1*time.Second).Should(BeTrue()) - - cancel() - }) - }) - Context("when running a standalone webhook", func() { - It("should reject create request for webhook that rejects all requests", func() { - ctx, cancel := context.WithCancel(context.Background()) - - By("generating the TLS config") - certPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.crt") - keyPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.key") - - certWatcher, err := certwatcher.New(certPath, keyPath) - Expect(err).NotTo(HaveOccurred()) - go func() { - Expect(certWatcher.Start(ctx)).NotTo(HaveOccurred()) - }() - - cfg := &tls.Config{ - NextProtos: []string{"h2"}, - GetCertificate: certWatcher.GetCertificate, - MinVersion: tls.VersionTLS12, - } - - By("generating the listener") - listener, err := tls.Listen("tcp", - net.JoinHostPort(testenv.WebhookInstallOptions.LocalServingHost, - strconv.Itoa(testenv.WebhookInstallOptions.LocalServingPort)), cfg) - Expect(err).NotTo(HaveOccurred()) - - By("creating and registering the standalone webhook") - hook, err := admission.StandaloneWebhook(admission.ValidatingWebhookFor( - &admissiontest.FakeValidator{ - ErrorToReturn: errors.New("Always denied"), - GVKToReturn: schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, - }), admission.StandaloneOptions{}) - Expect(err).NotTo(HaveOccurred()) - http.Handle("/failing", hook) - - By("running the http server") - srv := &http.Server{} - go func() { - idleConnsClosed := make(chan struct{}) - go func() { - <-ctx.Done() - Expect(srv.Shutdown(context.Background())).NotTo(HaveOccurred()) - close(idleConnsClosed) - }() - _ = srv.Serve(listener) - <-idleConnsClosed - }() - - Eventually(func() bool { - err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + err := c.Create(ctx, obj) + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) - - cancel() }) }) }) type rejectingValidator struct { - d *admission.Decoder -} - -func (v *rejectingValidator) InjectDecoder(d *admission.Decoder) error { - v.d = d - return nil + d admission.Decoder } func (v *rejectingValidator) Handle(ctx context.Context, req admission.Request) admission.Response { diff --git a/pkg/webhook/webhook_suite_test.go b/pkg/webhook/webhook_suite_test.go index b8ee879d36..ee9c1f4057 100644 --- a/pkg/webhook/webhook_suite_test.go +++ b/pkg/webhook/webhook_suite_test.go @@ -20,22 +20,20 @@ import ( "fmt" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Webhook Integration Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Webhook Integration Suite") } var testenv *envtest.Environment @@ -50,12 +48,12 @@ var _ = BeforeSuite(func() { var err error cfg, err = testenv.Start() Expect(err).NotTo(HaveOccurred()) -}, 60) +}) var _ = AfterSuite(func() { fmt.Println("stopping?") Expect(testenv.Stop()).To(Succeed()) -}, 60) +}) func initializeWebhookInEnvironment() { namespacedScopeV1 := admissionv1.NamespacedScope diff --git a/tools/setup-envtest/README.md b/tools/setup-envtest/README.md index 0a497c3ec4..a4de6f3eae 100644 --- a/tools/setup-envtest/README.md +++ b/tools/setup-envtest/README.md @@ -4,13 +4,19 @@ This is a small tool that manages binaries for envtest. It can be used to download new binaries, list currently installed and available ones, and clean up versions. -To use it, just go-install it on 1.16+ (it's a separate, self-contained +To use it, just go-install it with Golang 1.24+ (it's a separate, self-contained module): ```shell go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ``` +If you are using Golang 1.23, use the `release-0.20` branch instead: + +```shell +go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.20 +``` + For full documentation, run it with the `--help` flag, but here are some examples: @@ -40,27 +46,33 @@ setup-envtest use -i --use-env # sideload a pre-downloaded tarball as Kubernetes 1.16.2 into our store setup-envtest sideload 1.16.2 < downloaded-envtest.tar.gz + +# Per default envtest binaries are downloaded from: +# https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml +# To download from a custom index use the following: +setup-envtest use --index https://custom.com/envtest-releases.yaml + ``` ## Where does it put all those binaries? By default, binaries are stored in a subdirectory of an OS-specific data -directory, as per the OS's conventions. +directory, as per the OS's conventions. On Linux, this is `$XDG_DATA_HOME`; on Windows, `%LocalAppData`; and on OSX, `~/Library/Application Support`. There's an overall folder that holds all files, and inside that is -a folder for each version/platform pair. The exact directory structure is -not guarnateed, except that the leaf directory will contain the names -expected by envtest. You should always use `setup-envtest fetch` or +a folder for each version/platform pair. The exact directory structure is +not guaranteed, except that the leaf directory will contain the names +expected by envtest. You should always use `setup-envtest fetch` or `setup-envtest switch` (generally with the `-p path` or `-p env` flags) to get the directory that you should use. ## Why do I have to do that `source <(blah blah blah)` thing This is a normal binary, not a shell script, so we can't set the parent -process's environment variables. If you use this by hand a lot and want +process's environment variables. If you use this by hand a lot and want to save the typing, you could put something like the following in your `~/.zshrc` (or similar for bash/fish/whatever, modified to those): @@ -79,7 +91,7 @@ setup-envtest() { There are a few options. First, you'll probably want to set the `-i/--installed` flag. If you want -to avoid forgetting to set this flag, set the `ENVTEST_INSTALLED_ONLY` +to avoid forgetting to set this flag, set the `ENVTEST_INSTALLED_ONLY` env variable, which will switch that flag on by default. Then, you have a few options for managing your binaries: @@ -90,36 +102,18 @@ Then, you have a few options for managing your binaries: `--use-env` makes the command unconditionally use the value of KUBEBUILDER_ASSETS as long as it contains the required binaries, and - `-i` indicates that we only ever want to work with installed binaries - (no reaching out the the remote GCS storage). + `-i` indicates that we only ever want to work with installed binaries. As noted about, you can use `ENVTEST_INSTALLED_ONLY=true` to switch `-i` on by default, and you can use `ENVTEST_USE_ENV=true` to switch `--use-env` on by default. - If you want to use this tool, but download your gziped tarballs - separately, you can use the `sideload` command. You'll need to use the + separately, you can use the `sideload` command. You'll need to use the `-k/--version` flag to indicate which version you're sideloading. After that, it'll be as if you'd installed the binaries with `use`. -- If you want to talk to some internal source, you can use the - `--remote-bucket` and `--remote-server` options. The former sets which - GCS bucket to download from, and the latter sets the host to talk to as - if it were a GCS endpoint. Theoretically, you could use the latter - version to run an internal "mirror" -- the tool expects - - - `HOST/storage/v1/b/BUCKET/o` to return JSON like - - ```json - {"items": [ - {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}, - {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}, - ]} - ``` - - - `HOST/storage/v1/b/BUCKET/o/TARBALL_NAME` to return JSON like - `{"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}` - - - `HOST/storage/v1/b/BUCKET/o/TARBALL_NAME?alt=media` to return the - actual file contents +- If you want to talk to some internal source via HTTP, you can simply set `--index` + The index must contain references to envtest binary archives in the same format as: + https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml diff --git a/tools/setup-envtest/env/env.go b/tools/setup-envtest/env/env.go index e12a107352..6168739eb6 100644 --- a/tools/setup-envtest/env/env.go +++ b/tools/setup-envtest/env/env.go @@ -26,24 +26,24 @@ import ( // envtest binaries. // // In general, the methods will use the Exit{,Cause} functions from this package -// to indicate errors. Catch them with a `defer HandleExitWithCode()`. +// to indicate errors. Catch them with a `defer HandleExitWithCode()`. type Env struct { // the following *must* be set on input // Platform is our current platform Platform versions.PlatformItem - // VerifiySum indicates whether or not we should run checksums. + // VerifySum indicates whether we should run checksums. VerifySum bool - // NoDownload forces us to not contact GCS, looking only - // at local files instead. + // NoDownload forces us to not contact remote services, + // looking only at local files instead. NoDownload bool // ForceDownload forces us to ignore local files and always - // contact GCS & re-download. + // contact remote services & re-download. ForceDownload bool - // Client is our remote client for contacting GCS. - Client *remote.Client + // Client is our remote client for contacting remote services. + Client remote.Client // Log allows us to log. Log logr.Logger @@ -133,7 +133,7 @@ func (e *Env) ListVersions(ctx context.Context) { } // LatestVersion returns the latest version matching our version selector and -// platform from the remote server, with the correspoding checksum for later +// platform from the remote server, with the corresponding checksum for later // use as well. func (e *Env) LatestVersion(ctx context.Context) (versions.Concrete, versions.PlatformItem) { vers, err := e.Client.ListVersions(ctx) @@ -193,7 +193,7 @@ func (e *Env) ExistsAndValid() bool { // // If necessary, it will enumerate on-disk and remote versions to accomplish // this, finding a version that matches our version selector and platform. -// It will always yield a concrete version, it *may* yield a concrete platorm +// It will always yield a concrete version, it *may* yield a concrete platform // as well. func (e *Env) EnsureVersionIsSet(ctx context.Context) { if e.Version.AsConcrete() != nil { @@ -247,13 +247,13 @@ func (e *Env) EnsureVersionIsSet(ctx context.Context) { // if we're not forcing a download, and we have a newer local version, just use that if !e.ForceDownload && localVer != nil && localVer.NewerThan(serverVer) { - e.Platform.Platform = localPlat // update our data with md5 + e.Platform.Platform = localPlat // update our data with hash e.Version.MakeConcrete(*localVer) return } // otherwise, use the new version from the server - e.Platform = platform // update our data with md5 + e.Platform = platform // update our data with hash e.Version.MakeConcrete(serverVer) } @@ -266,13 +266,13 @@ func (e *Env) Fetch(ctx context.Context) { log := e.Log.WithName("fetch") // if we didn't just fetch it, grab the sum to verify - if e.VerifySum && e.Platform.MD5 == "" { + if e.VerifySum && e.Platform.Hash == nil { if err := e.Client.FetchSum(ctx, *e.Version.AsConcrete(), &e.Platform); err != nil { - ExitCause(2, err, "unable to fetch checksum for requested version") + ExitCause(2, err, "unable to fetch hash for requested version") } } if !e.VerifySum { - e.Platform.MD5 = "" // skip verification + e.Platform.Hash = nil // skip verification } var packedPath string @@ -365,8 +365,8 @@ func (e *Env) PrintInfo(printFmt PrintFormat) { case PrintOverview: fmt.Fprintf(e.Out, "Version: %s\n", e.Version) fmt.Fprintf(e.Out, "OS/Arch: %s\n", e.Platform) - if e.Platform.MD5 != "" { - fmt.Fprintf(e.Out, "md5: %s\n", e.Platform.MD5) + if e.Platform.Hash != nil { + fmt.Fprintf(e.Out, "%s: %s\n", e.Platform.Hash.Type, e.Platform.Hash.Value) } fmt.Fprintf(e.Out, "Path: %s\n", path) case PrintPath: @@ -409,7 +409,7 @@ func (e *Env) Sideload(ctx context.Context, input io.Reader) { } var ( - // expectedExectuables are the executables that are checked in PathMatches + // expectedExecutables are the executables that are checked in PathMatches // for non-store paths. expectedExecutables = []string{ "kube-apiserver", @@ -458,7 +458,7 @@ func (e *Env) PathMatches(value string) bool { } // versionFromPathName checks if the given path's last component looks like one -// of our versions, and, if so, what version it represents. If succesfull, +// of our versions, and, if so, what version it represents. If successful, // it'll set version and platform, and return true. Otherwise it returns // false. func (e *Env) versionFromPathName(value string) bool { diff --git a/tools/setup-envtest/env/env_suite_test.go b/tools/setup-envtest/env/env_suite_test.go index 7d9fe9c179..3400dd91aa 100644 --- a/tools/setup-envtest/env/env_suite_test.go +++ b/tools/setup-envtest/env/env_suite_test.go @@ -19,7 +19,7 @@ package env_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/go-logr/logr" diff --git a/tools/setup-envtest/env/env_test.go b/tools/setup-envtest/env/env_test.go index 874ac7a736..fd6e7633bd 100644 --- a/tools/setup-envtest/env/env_test.go +++ b/tools/setup-envtest/env/env_test.go @@ -19,7 +19,7 @@ package env_test import ( "bytes" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/spf13/afero" diff --git a/tools/setup-envtest/go.mod b/tools/setup-envtest/go.mod index be890a0c96..15c64f8b57 100644 --- a/tools/setup-envtest/go.mod +++ b/tools/setup-envtest/go.mod @@ -1,13 +1,29 @@ module sigs.k8s.io/controller-runtime/tools/setup-envtest -go 1.16 +go 1.24.0 require ( - github.com/go-logr/logr v0.4.0 - github.com/go-logr/zapr v0.4.0 - github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.15.0 - github.com/spf13/afero v1.6.0 - github.com/spf13/pflag v1.0.5 - go.uber.org/zap v1.19.0 + github.com/go-logr/logr v1.4.2 + github.com/go-logr/zapr v1.3.0 + github.com/onsi/ginkgo/v2 v2.22.2 + github.com/onsi/gomega v1.36.2 + github.com/spf13/afero v1.12.0 + github.com/spf13/pflag v1.0.6 + go.uber.org/zap v1.27.0 + k8s.io/apimachinery v0.34.1 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/tools v0.28.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/setup-envtest/go.sum b/tools/setup-envtest/go.sum index ec17d1e4a1..dfc8e7cce2 100644 --- a/tools/setup-envtest/go.sum +++ b/tools/setup-envtest/go.sum @@ -1,140 +1,52 @@ -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/tools/setup-envtest/main.go b/tools/setup-envtest/main.go index 9c0ab78057..7eb5ec43d3 100644 --- a/tools/setup-envtest/main.go +++ b/tools/setup-envtest/main.go @@ -49,10 +49,8 @@ var ( binDir = flag.String("bin-dir", "", "directory to store binary assets (default: $OS_SPECIFIC_DATA_DIR/envtest-binaries)") - remoteBucket = flag.String("remote-bucket", "kubebuilder-tools", "remote GCS bucket to download from") - remoteServer = flag.String("remote-server", "storage.googleapis.com", - "remote server to query from. You can override this if you want to run "+ - "an internal storage server instead, or for testing.") + + index = flag.String("index", remote.DefaultIndexURL, "index to discover envtest binaries") ) // TODO(directxman12): handle interrupts? @@ -81,13 +79,15 @@ func setupEnv(globalLog logr.Logger, version string) *envp.Env { } log.V(1).Info("using binaries directory", "dir", *binDir) + client := &remote.HTTPClient{ + Log: globalLog.WithName("storage-client"), + IndexURL: *index, + } + log.V(1).Info("using HTTP client", "index", *index) + env := &envp.Env{ - Log: globalLog, - Client: &remote.Client{ - Log: globalLog.WithName("storage-client"), - Bucket: *remoteBucket, - Server: *remoteServer, - }, + Log: globalLog, + Client: client, VerifySum: *verify, ForceDownload: *force, NoDownload: *installedOnly, @@ -169,7 +169,7 @@ Commands: use: get information for the requested version, downloading it if necessary and allowed. - Needs a concrete platform (no wildcards), but wilcard versions are supported. + Needs a concrete platform (no wildcards), but wildcard versions are supported. list: list installed *and* available versions matching the given version & platform. @@ -184,6 +184,9 @@ Commands: reads a .tar.gz file from stdin and expand it into the store. must have a concrete version and platform. + version: + list the installed version of setup-envtest. + Versions: Versions take the form of a small subset of semver selectors. @@ -192,7 +195,7 @@ Versions: Z may also be '*' or 'x' to match a wildcard. You may also just write X.Y, which means X.Y.*. - A version may be prefixed with '~' to match the the most recent Z release + A version may be prefixed with '~' to match the most recent Z release in the given Y release ( [X.Y.Z, X.Y+1.0) ). Finally, you may suffix the version with '!' to force checking the @@ -203,7 +206,7 @@ Versions: 1.16.x / 1.16.* / 1.16 # any 1.16 version ~1.19.3 # any 1.19 version that's at least 1.19.3 <1.17 # any release 1.17.x or below - 1.22.x! # the latest one 1.22 release avaible remotely + 1.22.x! # the latest one 1.22 release available remotely Output: @@ -256,7 +259,6 @@ Environment Variables: version = flag.Arg(1) } env := setupEnv(globalLog, version) - // perform our main set of actions switch action := flag.Arg(0); action { case "use": @@ -274,6 +276,8 @@ Environment Variables: Input: os.Stdin, PrintFormat: printFormat, }.Do(env) + case "version": + workflows.Version{}.Do(env) default: flag.Usage() envp.Exit(2, "unknown action %q", action) diff --git a/tools/setup-envtest/remote/client.go b/tools/setup-envtest/remote/client.go index 00e4840813..24efd6daff 100644 --- a/tools/setup-envtest/remote/client.go +++ b/tools/setup-envtest/remote/client.go @@ -1,218 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 -// Copyright 2021 The Kubernetes Authors +// Copyright 2024 The Kubernetes Authors package remote import ( "context" - "crypto/md5" //nolint:gosec - "encoding/base64" - "encoding/json" - "fmt" "io" - "net/http" - "net/url" - "path" - "sort" - "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" ) -// objectList is the parts we need of the GCS "list-objects-in-bucket" endpoint. -type objectList struct { - Items []bucketObject `json:"items"` - NextPageToken string `json:"nextPageToken"` -} - -// bucketObject is the parts we need of the GCS object metadata. -type bucketObject struct { - Name string `json:"name"` - Hash string `json:"md5Hash"` -} - -// Client is a basic client for fetching versions of the envtest binary archives -// from GCS. -type Client struct { - // Bucket is the bucket to fetch from. - Bucket string - - // Server is the GCS-like storage server - Server string - - // Log allows us to log. - Log logr.Logger - - // Insecure uses http for testing - Insecure bool -} - -func (c *Client) scheme() string { - if c.Insecure { - return "http" - } - return "https" -} - -// ListVersions lists all available tools versions in the given bucket, along -// with supported os/arch combos and the corresponding hash. -// -// The results are sorted with newer versions first. -func (c *Client) ListVersions(ctx context.Context) ([]versions.Set, error) { - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o"), - } - query := make(url.Values) - - knownVersions := map[versions.Concrete][]versions.PlatformItem{} - for cont := true; cont; { - c.Log.V(1).Info("listing bucket to get versions", "bucket", c.Bucket) - - loc.RawQuery = query.Encode() - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return nil, fmt.Errorf("unable to construct request to list bucket items: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("unable to perform request to list bucket items: %w", err) - } - - err = func() error { - defer resp.Body.Close() - if resp.StatusCode != 200 { - return fmt.Errorf("unable list bucket items -- got status %q from GCS", resp.Status) - } - - var list objectList - if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { - return fmt.Errorf("unable unmarshal bucket items list: %w", err) - } - - // continue listing if needed - cont = list.NextPageToken != "" - query.Set("pageToken", list.NextPageToken) - - for _, item := range list.Items { - ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, item.Name) - if ver == nil { - c.Log.V(1).Info("skipping bucket object -- does not appear to be a versioned tools object", "name", item.Name) - continue - } - c.Log.V(1).Info("found version", "version", ver, "platform", details) - knownVersions[*ver] = append(knownVersions[*ver], versions.PlatformItem{ - Platform: details, - MD5: item.Hash, - }) - } - - return nil - }() - if err != nil { - return nil, err - } - } - - res := make([]versions.Set, 0, len(knownVersions)) - for ver, details := range knownVersions { - res = append(res, versions.Set{Version: ver, Platforms: details}) - } - // sort in inverse order so that the newest one is first - sort.Slice(res, func(i, j int) bool { - first, second := res[i].Version, res[j].Version - return first.NewerThan(second) - }) - - return res, nil -} - -// GetVersion downloads the given concrete version for the given concrete platform, writing it to the out. -func (c *Client) GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error { - itemName := platform.ArchiveName(version) - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), - RawQuery: "alt=media", - } - - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return fmt.Errorf("unable to construct request to fetch %s: %w", itemName, err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("unable to fetch %s (%s): %w", itemName, req.URL, err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("unable fetch %s (%s) -- got status %q from GCS", itemName, req.URL, resp.Status) - } - - if platform.MD5 != "" { - // stream in chunks to do the checksum, don't load the whole thing into - // memory to avoid causing issues with big files. - buf := make([]byte, 32*1024) // 32KiB, same as io.Copy - checksum := md5.New() //nolint:gosec - for cont := true; cont; { - amt, err := resp.Body.Read(buf) - if err != nil && err != io.EOF { - return fmt.Errorf("unable read next chunk of %s: %w", itemName, err) - } - if amt > 0 { - // checksum never returns errors according to docs - checksum.Write(buf[:amt]) - if _, err := out.Write(buf[:amt]); err != nil { - return fmt.Errorf("unable write next chunk of %s: %w", itemName, err) - } - } - cont = amt > 0 && err != io.EOF - } - - sum := base64.StdEncoding.EncodeToString(checksum.Sum(nil)) - - if sum != platform.MD5 { - return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (reported from GCS)", itemName, sum, platform.MD5) - } - } else if _, err := io.Copy(out, resp.Body); err != nil { - return fmt.Errorf("unable to download %s: %w", itemName, err) - } - return nil -} - -// FetchSum fetches the checksum for the given concrete version & platform into -// the given platform item. -func (c *Client) FetchSum(ctx context.Context, ver versions.Concrete, pl *versions.PlatformItem) error { - itemName := pl.ArchiveName(ver) - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), - } - - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return fmt.Errorf("unable to construct request to fetch metadata for %s: %w", itemName, err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("unable to fetch metadata for %s: %w", itemName, err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("unable fetch metadata for %s -- got status %q from GCS", itemName, resp.Status) - } +// Client is an interface to get and list envtest binary archives. +type Client interface { + ListVersions(ctx context.Context) ([]versions.Set, error) - var item bucketObject - if err := json.NewDecoder(resp.Body).Decode(&item); err != nil { - return fmt.Errorf("unable to unmarshal metadata for %s: %w", itemName, err) - } + GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error - pl.MD5 = item.Hash - return nil + FetchSum(ctx context.Context, ver versions.Concrete, pl *versions.PlatformItem) error } diff --git a/tools/setup-envtest/remote/http_client.go b/tools/setup-envtest/remote/http_client.go new file mode 100644 index 0000000000..0339654a82 --- /dev/null +++ b/tools/setup-envtest/remote/http_client.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 The Kubernetes Authors + +package remote + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "sort" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" + "sigs.k8s.io/yaml" +) + +// DefaultIndexURL is the default index used in HTTPClient. +var DefaultIndexURL = "https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml" + +var _ Client = &HTTPClient{} + +// HTTPClient is a client for fetching versions of the envtest binary archives +// from an index via HTTP. +type HTTPClient struct { + // Log allows us to log. + Log logr.Logger + + // IndexURL is the URL of the index, defaults to DefaultIndexURL. + IndexURL string +} + +// Index represents an index of envtest binary archives. Example: +// +// releases: +// v1.28.0: +// envtest-v1.28.0-darwin-amd64.tar.gz: +// hash: +// selfLink: +type Index struct { + // Releases maps Kubernetes versions to Releases (envtest archives). + Releases map[string]Release `json:"releases"` +} + +// Release maps an archive name to an archive. +type Release map[string]Archive + +// Archive contains the self link to an archive and its hash. +type Archive struct { + Hash string `json:"hash"` + SelfLink string `json:"selfLink"` +} + +// ListVersions lists all available tools versions in the index, along +// with supported os/arch combos and the corresponding hash. +// +// The results are sorted with newer versions first. +func (c *HTTPClient) ListVersions(ctx context.Context) ([]versions.Set, error) { + index, err := c.getIndex(ctx) + if err != nil { + return nil, err + } + + knownVersions := map[versions.Concrete][]versions.PlatformItem{} + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + c.Log.V(1).Info("found version", "version", ver, "platform", details) + knownVersions[*ver] = append(knownVersions[*ver], versions.PlatformItem{ + Platform: details, + Hash: &versions.Hash{ + Type: versions.SHA512HashType, + Encoding: versions.HexHashEncoding, + Value: archive.Hash, + }, + }) + } + } + + res := make([]versions.Set, 0, len(knownVersions)) + for ver, details := range knownVersions { + res = append(res, versions.Set{Version: ver, Platforms: details}) + } + // sort in inverse order so that the newest one is first + sort.Slice(res, func(i, j int) bool { + first, second := res[i].Version, res[j].Version + return first.NewerThan(second) + }) + + return res, nil +} + +// GetVersion downloads the given concrete version for the given concrete platform, writing it to the out. +func (c *HTTPClient) GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error { + index, err := c.getIndex(ctx) + if err != nil { + return err + } + + var loc *url.URL + var name string + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + + if *ver == version && details.OS == platform.OS && details.Arch == platform.Arch { + loc, err = url.Parse(archive.SelfLink) + if err != nil { + return fmt.Errorf("error parsing selfLink %q, %w", loc, err) + } + name = archiveName + break + } + } + } + if name == "" { + return fmt.Errorf("unable to find archive for %s (%s,%s)", version, platform.OS, platform.Arch) + } + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return fmt.Errorf("unable to construct request to fetch %s: %w", name, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("unable to fetch %s (%s): %w", name, req.URL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("unable fetch %s (%s) -- got status %q", name, req.URL, resp.Status) + } + + return readBody(resp, out, name, platform) +} + +// FetchSum fetches the checksum for the given concrete version & platform into +// the given platform item. +func (c *HTTPClient) FetchSum(ctx context.Context, version versions.Concrete, platform *versions.PlatformItem) error { + index, err := c.getIndex(ctx) + if err != nil { + return err + } + + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + + if *ver == version && details.OS == platform.OS && details.Arch == platform.Arch { + platform.Hash = &versions.Hash{ + Type: versions.SHA512HashType, + Encoding: versions.HexHashEncoding, + Value: archive.Hash, + } + return nil + } + } + } + + return fmt.Errorf("unable to find archive for %s (%s,%s)", version, platform.OS, platform.Arch) +} + +func (c *HTTPClient) getIndex(ctx context.Context) (*Index, error) { + indexURL := c.IndexURL + if indexURL == "" { + indexURL = DefaultIndexURL + } + + loc, err := url.Parse(indexURL) + if err != nil { + return nil, fmt.Errorf("unable to parse index URL: %w", err) + } + + c.Log.V(1).Info("listing versions", "index", indexURL) + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return nil, fmt.Errorf("unable to construct request to get index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("unable to perform request to get index: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unable to get index -- got status %q", resp.Status) + } + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to get index -- unable to read body %w", err) + } + + var index Index + if err := yaml.Unmarshal(responseBody, &index); err != nil { + return nil, fmt.Errorf("unable to unmarshal index: %w", err) + } + return &index, nil +} diff --git a/tools/setup-envtest/remote/read_body.go b/tools/setup-envtest/remote/read_body.go new file mode 100644 index 0000000000..1c71102897 --- /dev/null +++ b/tools/setup-envtest/remote/read_body.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2024 The Kubernetes Authors + +package remote + +import ( + "crypto/md5" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + + "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" +) + +func readBody(resp *http.Response, out io.Writer, archiveName string, platform versions.PlatformItem) error { + if platform.Hash != nil { + // stream in chunks to do the checksum, don't load the whole thing into + // memory to avoid causing issues with big files. + buf := make([]byte, 32*1024) // 32KiB, same as io.Copy + var hasher hash.Hash + switch platform.Hash.Type { + case versions.SHA512HashType: + hasher = sha512.New() + case versions.MD5HashType: + hasher = md5.New() + default: + return fmt.Errorf("hash type %s not implemented", platform.Hash.Type) + } + for cont := true; cont; { + amt, err := resp.Body.Read(buf) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("unable read next chunk of %s: %w", archiveName, err) + } + if amt > 0 { + // checksum never returns errors according to docs + hasher.Write(buf[:amt]) + if _, err := out.Write(buf[:amt]); err != nil { + return fmt.Errorf("unable write next chunk of %s: %w", archiveName, err) + } + } + cont = amt > 0 && !errors.Is(err, io.EOF) + } + + var sum string + switch platform.Hash.Encoding { + case versions.Base64HashEncoding: + sum = base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + case versions.HexHashEncoding: + sum = hex.EncodeToString(hasher.Sum(nil)) + default: + return fmt.Errorf("hash encoding %s not implemented", platform.Hash.Encoding) + } + if sum != platform.Hash.Value { + return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (reported)", archiveName, sum, platform.Hash.Value) + } + } else if _, err := io.Copy(out, resp.Body); err != nil { + return fmt.Errorf("unable to download %s: %w", archiveName, err) + } + return nil +} diff --git a/tools/setup-envtest/store/store.go b/tools/setup-envtest/store/store.go index e3c9ca018f..bb5a1f7bcd 100644 --- a/tools/setup-envtest/store/store.go +++ b/tools/setup-envtest/store/store.go @@ -38,7 +38,7 @@ func (i Item) String() string { } // Filter is a version spec & platform selector (i.e. platform -// potentially with wilcards) to filter store items. +// potentially with wildcards) to filter store items. type Filter struct { Version versions.Spec Platform versions.Platform @@ -68,7 +68,12 @@ func NewAt(path string) *Store { // Initialize ensures that the store is all set up on disk, etc. func (s *Store) Initialize(ctx context.Context) error { - logr.FromContext(ctx).V(1).Info("ensuring base binaries dir exists") + log, err := logr.FromContext(ctx) + if err != nil { + return err + } + + log.V(1).Info("ensuring base binaries dir exists") if err := s.unpackedBase().MkdirAll("", 0755); err != nil { return fmt.Errorf("unable to make sure base binaries dir exists: %w", err) } @@ -109,7 +114,11 @@ func (s *Store) List(ctx context.Context, matching Filter) ([]Item, error) { // Add adds this item to the store, with the given contents (a .tar.gz file). func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr error) { - log := logr.FromContext(ctx) + log, err := logr.FromContext(ctx) + if err != nil { + return err + } + itemName := item.dirName() log = log.WithValues("version-platform", itemName) itemPath := s.unpackedPath(itemName) @@ -126,7 +135,7 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr }() log.V(1).Info("ensuring version-platform binaries dir exists and is empty & writable") - _, err := itemPath.Stat("") + _, err = itemPath.Stat("") if err != nil && !errors.Is(err, afero.ErrFileNotFound) { return fmt.Errorf("unable to ensure version-platform binaries dir %s exists", itemName) } @@ -165,7 +174,7 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr } if err := func() error { // IIFE to get the defer properly in a loop defer binOut.Close() - if _, err := io.Copy(binOut, tarReader); err != nil { //nolint:gosec + if _, err := io.Copy(binOut, tarReader); err != nil { return fmt.Errorf("unable to write file %s from archive to disk for version-platform pair %s", targetPath, itemName) } return nil @@ -173,7 +182,7 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr return err } } - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { //nolint:govet return fmt.Errorf("unable to finish un-tar-ing the downloaded archive: %w", err) } log.V(1).Info("unpacked archive") @@ -191,7 +200,11 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr // It returns a list of the successfully removed items (even in the case // of an error). func (s *Store) Remove(ctx context.Context, matching Filter) ([]Item, error) { - log := logr.FromContext(ctx) + log, err := logr.FromContext(ctx) + if err != nil { + return nil, err + } + var removed []Item var savedErr error if err := s.eachItem(ctx, matching, func(name string, item Item) { @@ -216,7 +229,7 @@ func (s *Store) Remove(ctx context.Context, matching Filter) ([]Item, error) { func (s *Store) Path(item Item) (string, error) { path := s.unpackedPath(item.dirName()) // NB(directxman12): we need root's realpath because RealPath only - // looks at it's own path, and so thus doesn't prepend the underlying + // looks at its own path, and so thus doesn't prepend the underlying // root's base path. // // Technically, if we're fed something that's double wrapped as root, @@ -237,7 +250,11 @@ func (s *Store) unpackedPath(name string) afero.Fs { // eachItem iterates through the on-disk versions that match our version & platform selector, // calling the callback for each. func (s *Store) eachItem(ctx context.Context, filter Filter, cb func(name string, item Item)) error { - log := logr.FromContext(ctx) + log, err := logr.FromContext(ctx) + if err != nil { + return err + } + entries, err := afero.ReadDir(s.unpackedBase(), "") if err != nil { return fmt.Errorf("unable to list folders in store's unpacked directory: %w", err) diff --git a/tools/setup-envtest/store/store_suite_test.go b/tools/setup-envtest/store/store_suite_test.go index 2eb909af6b..649c22d545 100644 --- a/tools/setup-envtest/store/store_suite_test.go +++ b/tools/setup-envtest/store/store_suite_test.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -40,8 +40,8 @@ func zapLogger() logr.Logger { return zapr.NewLogger(zapLog) } -func logCtx() context.Context { - return logr.NewContext(context.Background(), testLog) +func logCtx(ctx context.Context) context.Context { + return logr.NewContext(ctx, testLog) } func TestStore(t *testing.T) { diff --git a/tools/setup-envtest/store/store_test.go b/tools/setup-envtest/store/store_test.go index 862996abfa..575d49dd3b 100644 --- a/tools/setup-envtest/store/store_test.go +++ b/tools/setup-envtest/store/store_test.go @@ -20,12 +20,12 @@ import ( "archive/tar" "bytes" "compress/gzip" + "crypto/rand" "io" "io/fs" - "math/rand" "path/filepath" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/spf13/afero" @@ -47,23 +47,23 @@ var _ = Describe("Store", func() { } }) Describe("initialization", func() { - It("should ensure the repo root exists", func() { + It("should ensure the repo root exists", func(ctx SpecContext) { // remove the old dir Expect(st.Root.RemoveAll("")).To(Succeed(), "should be able to remove the store before trying to initialize") - Expect(st.Initialize(logCtx())).To(Succeed(), "initialization should succeed") + Expect(st.Initialize(logCtx(ctx))).To(Succeed(), "initialization should succeed") Expect(st.Root.Stat("k8s")).NotTo(BeNil(), "store's binary dir should exist") }) - It("should be fine if the repo root already exists", func() { - Expect(st.Initialize(logCtx())).To(Succeed()) + It("should be fine if the repo root already exists", func(ctx SpecContext) { + Expect(st.Initialize(logCtx(ctx))).To(Succeed()) }) }) Describe("listing items", func() { - It("should filter results by the given filter, sorted in version order (newest first)", func() { + It("should filter results by the given filter, sorted in version order (newest first)", func(ctx SpecContext) { sel, err := versions.FromExpr("<=1.16") Expect(err).NotTo(HaveOccurred(), "should be able to construct <=1.16 selector") - Expect(st.List(logCtx(), store.Filter{ + Expect(st.List(logCtx(ctx), store.Filter{ Version: sel, Platform: versions.Platform{OS: "*", Arch: "amd64"}, })).To(Equal([]store.Item{ @@ -73,16 +73,16 @@ var _ = Describe("Store", func() { {Version: ver(1, 14, 26), Platform: versions.Platform{OS: "linux", Arch: "amd64"}}, })) }) - It("should skip non-folders in the store", func() { + It("should skip non-folders in the store", func(ctx SpecContext) { Expect(afero.WriteFile(st.Root, "k8s/2.3.6-linux-amd128", []byte{0x01}, fs.ModePerm)).To(Succeed(), "should be able to create a non-store file in the store directory") - Expect(st.List(logCtx(), store.Filter{ + Expect(st.List(logCtx(ctx), store.Filter{ Version: versions.AnyVersion, Platform: versions.Platform{OS: "linux", Arch: "amd128"}, })).To(BeEmpty()) }) - It("should skip non-matching names in the store", func() { + It("should skip non-matching names in the store", func(ctx SpecContext) { Expect(st.Root.Mkdir("k8s/somedir-2.3.6-linux-amd128", fs.ModePerm)).To(Succeed(), "should be able to create a non-store file in the store directory") - Expect(st.List(logCtx(), store.Filter{ + Expect(st.List(logCtx(ctx), store.Filter{ Version: versions.AnyVersion, Platform: versions.Platform{OS: "linux", Arch: "amd128"}, })).To(BeEmpty()) }) @@ -90,10 +90,10 @@ var _ = Describe("Store", func() { Describe("removing items", func() { var res []store.Item - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { sel, err := versions.FromExpr("<=1.16") Expect(err).NotTo(HaveOccurred(), "should be able to construct <=1.16 selector") - res, err = st.Remove(logCtx(), store.Filter{ + res, err = st.Remove(logCtx(ctx), store.Filter{ Version: sel, Platform: versions.Platform{OS: "*", Arch: "amd64"}, }) @@ -125,28 +125,30 @@ var _ = Describe("Store", func() { }) }) - Describe("adding items", func() { - It("should support .tar.gz input", func() { - Expect(st.Add(logCtx(), newItem, makeFakeArchive(newName))).To(Succeed()) + Describe("adding items (controller-tools archives)", func() { + archiveName := "envtest-v1.16.3-linux-amd64.tar.gz" + + It("should support .tar.gz input", func(ctx SpecContext) { + Expect(st.Add(logCtx(ctx), newItem, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) Expect(st.Has(newItem)).To(BeTrue(), "should have the item after adding it") }) - It("should extract binaries from the given archive to a directly to the item's directory, regardless of path", func() { - Expect(st.Add(logCtx(), newItem, makeFakeArchive(newName))).To(Succeed()) + It("should extract binaries from the given archive to a directly to the item's directory, regardless of path", func(ctx SpecContext) { + Expect(st.Add(logCtx(ctx), newItem, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) dirName := newItem.Platform.BaseName(newItem.Version) - Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "some-file"))).To(HavePrefix(newName + "some-file")) - Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "other-file"))).To(HavePrefix(newName + "other-file")) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "some-file"))).To(HavePrefix(archiveName + "some-file")) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "other-file"))).To(HavePrefix(archiveName + "other-file")) }) - It("should clean up any existing item directory before creating the new one", func() { + It("should clean up any existing item directory before creating the new one", func(ctx SpecContext) { item := localVersions[0] - Expect(st.Add(logCtx(), item, makeFakeArchive(newName))).To(Succeed()) + Expect(st.Add(logCtx(ctx), item, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) Expect(st.Root.Stat(filepath.Join("k8s", item.Platform.BaseName(item.Version)))).NotTo(BeNil(), "new files should exist") }) - It("should clean up if it errors before finishing", func() { + It("should clean up if it errors before finishing", func(ctx SpecContext) { item := localVersions[0] - Expect(st.Add(logCtx(), item, new(bytes.Buffer))).NotTo(Succeed(), "should fail to extract") + Expect(st.Add(logCtx(ctx), item, new(bytes.Buffer))).NotTo(Succeed(), "should fail to extract") _, err := st.Root.Stat(filepath.Join("k8s", item.Platform.BaseName(item.Version))) Expect(err).To(HaveOccurred(), "the binaries dir for the item should be gone") @@ -187,8 +189,6 @@ var ( Version: ver(1, 16, 3), Platform: versions.Platform{OS: "linux", Arch: "amd64"}, } - - newName = "kubebuilder-tools-1.16.3-linux-amd64.tar.gz" ) func ver(major, minor, patch int) versions.Concrete { @@ -199,13 +199,13 @@ func ver(major, minor, patch int) versions.Concrete { } } -func makeFakeArchive(magic string) io.Reader { +func makeFakeArchive(magic, relativePath string) io.Reader { out := new(bytes.Buffer) gzipWriter := gzip.NewWriter(out) tarWriter := tar.NewWriter(gzipWriter) Expect(tarWriter.WriteHeader(&tar.Header{ Typeflag: tar.TypeDir, - Name: "kubebuilder/bin/", // so we can ensure we skip non-files + Name: relativePath, // so we can ensure we skip non-files Mode: 0777, })).To(Succeed()) for _, fileName := range []string{"some-file", "other-file"} { @@ -214,13 +214,13 @@ func makeFakeArchive(magic string) io.Reader { copy(chunk[:], magic) copy(chunk[len(magic):], fileName) start := len(magic) + len(fileName) - if _, err := rand.Read(chunk[start:]); err != nil { //nolint:gosec + if _, err := rand.Read(chunk[start:]); err != nil { panic(err) } - // write to kubebuilder/bin/fileName + // write to relativePath/fileName err := tarWriter.WriteHeader(&tar.Header{ - Name: "kubebuilder/bin/" + fileName, + Name: relativePath + fileName, Size: int64(len(chunk[:])), Mode: 0777, // so we can check that we fix this later }) diff --git a/tools/setup-envtest/version/version.go b/tools/setup-envtest/version/version.go new file mode 100644 index 0000000000..1d148b085d --- /dev/null +++ b/tools/setup-envtest/version/version.go @@ -0,0 +1,21 @@ +package version + +import "runtime/debug" + +// Version to be set using ldflags: +// -ldflags "-X sigs.k8s.io/controller-runtime/tools/setup-envtest/version.version=v1.0.0" +// falls back to module information is unse +var version = "" + +// Version returns the version of the main module +func Version() string { + if version != "" { + return version + } + info, ok := debug.ReadBuildInfo() + if !ok || info == nil || info.Main.Version == "" { + // binary has not been built with module support or doesn't contain a version. + return "(unknown)" + } + return info.Main.Version +} diff --git a/pkg/config/v1alpha1/doc.go b/tools/setup-envtest/version/version_suite_test.go similarity index 69% rename from pkg/config/v1alpha1/doc.go rename to tools/setup-envtest/version/version_suite_test.go index 1e3adbafb8..99c623e8d4 100644 --- a/pkg/config/v1alpha1/doc.go +++ b/tools/setup-envtest/version/version_suite_test.go @@ -1,10 +1,8 @@ /* -Copyright 2020 The Kubernetes Authors. - +Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software @@ -14,7 +12,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package v1alpha1 provides the ControllerManagerConfiguration used for -// configuring ctrl.Manager -// +kubebuilder:object:generate=true -package v1alpha1 +package version + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestVersioning(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Test Version Suite") +} diff --git a/tools/setup-envtest/version/version_test.go b/tools/setup-envtest/version/version_test.go new file mode 100644 index 0000000000..4178cac870 --- /dev/null +++ b/tools/setup-envtest/version/version_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and + +limitations under the License. +*/ + +package version + +import ( + "runtime/debug" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("TestVersion", func() { + + info, ok := debug.ReadBuildInfo() + Expect(ok).To(BeTrue()) + tests := map[string]struct { + version string + expected string + }{ + "empty returns build info": { + version: "", + expected: info.Main.Version, + }, + "set to a value returns it": { + version: "1.2.3", + expected: "1.2.3", + }, + } + for name, tc := range tests { + It("Version set to "+name, func() { + versionBackup := version + defer func() { + version = versionBackup + }() + version = tc.version + result := Version() + Expect(result).To(Equal(tc.expected)) + }) + } +}) diff --git a/tools/setup-envtest/versions/misc_test.go b/tools/setup-envtest/versions/misc_test.go index 3429211f27..a609f4dc60 100644 --- a/tools/setup-envtest/versions/misc_test.go +++ b/tools/setup-envtest/versions/misc_test.go @@ -17,7 +17,7 @@ limitations under the License. package versions_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" @@ -95,7 +95,7 @@ var _ = Describe("Platform", func() { Specify("knows how to produce an archive name", func() { plat := Platform{OS: "linux", Arch: "amd64"} ver := Concrete{Major: 1, Minor: 16, Patch: 3} - Expect(plat.ArchiveName(ver)).To(Equal("kubebuilder-tools-1.16.3-linux-amd64.tar.gz")) + Expect(plat.ArchiveName(ver)).To(Equal("envtest-v1.16.3-linux-amd64.tar.gz")) }) Describe("parsing", func() { @@ -110,14 +110,14 @@ var _ = Describe("Platform", func() { Expect(ver).To(BeNil()) }) }) - Context("for archive names", func() { - It("should accept strings of the form kubebuilder-tools-x.y.z-os-arch.tar.gz", func() { - ver, plat := ExtractWithPlatform(ArchiveRE, "kubebuilder-tools-1.16.3-linux-amd64.tar.gz") + Context("for archive names (controller-tools)", func() { + It("should accept strings of the form envtest-vx.y.z-os-arch.tar.gz", func() { + ver, plat := ExtractWithPlatform(ArchiveRE, "envtest-v1.16.3-linux-amd64.tar.gz") Expect(ver).To(Equal(&Concrete{Major: 1, Minor: 16, Patch: 3})) Expect(plat).To(Equal(Platform{OS: "linux", Arch: "amd64"})) }) It("should reject nonsense strings", func() { - ver, _ := ExtractWithPlatform(ArchiveRE, "kubebuilder-tools-1.16.3-linux-amd64.tar.sum") + ver, _ := ExtractWithPlatform(ArchiveRE, "envtest-v1.16.3-linux-amd64.tar.sum") Expect(ver).To(BeNil()) }) }) diff --git a/tools/setup-envtest/versions/parse.go b/tools/setup-envtest/versions/parse.go index 4b3c5bb5ba..cd25710b2b 100644 --- a/tools/setup-envtest/versions/parse.go +++ b/tools/setup-envtest/versions/parse.go @@ -17,23 +17,20 @@ var ( // ConcreteVersionRE matches a concrete version anywhere in the string. ConcreteVersionRE = regexp.MustCompile(`(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)`) - // OnlyConcreteVersionRE matches a string that's just a concrete version. - OnlyConcreteVersionRE = regexp.MustCompile(`^` + ConcreteVersionRE.String() + `$`) ) // FromExpr extracts a version from a string in the form of a semver version, // where X, Y, and Z may also be wildcards ('*', 'x'), // and pre-release names & numbers may also be wildcards. The prerelease section is slightly // restricted to match what k8s does. -// The the whole string is a version selector as follows: -// -// - X.Y.Z matches version X.Y.Z where x, y, and z are -// are ints >= 0, and Z may be '*' or 'x' -// - X.Y is equivalent to X.Y.* -// - ~X.Y.Z means >= X.Y.Z && < X.Y+1.0 -// - = 0, and Z may be '*' or 'x' +// - X.Y is equivalent to X.Y.* +// - ~X.Y.Z means >= X.Y.Z && < X.Y+1.0 +// -