diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f3eba9f073..9d68a39241 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,36 +1,18 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - version: 2 updates: - - # Maintain dependencies for GitHub Actions - - package-ecosystem: "github-actions" - # Workflow files stored in the - # default location of `.github/workflows` - directory: "/" - schedule: - interval: "weekly" - commit-message: - prefix: ":seedling:" - labels: - - "ok-to-test" - - # Maintain dependencies for go - - package-ecosystem: "gomod" - directory: "/" - schedule: - interval: "weekly" - commit-message: - prefix: ":seedling:" - # Ignore K8 packages as these are done manually - ignore: - - dependency-name: "k8s.io/api" - - dependency-name: "k8s.io/apiextensions-apiserver" - - dependency-name: "k8s.io/apimachinery" - - dependency-name: "k8s.io/client-go" - - dependency-name: "k8s.io/component-base" - labels: - - "ok-to-test" +# GitHub Actions +- package-ecosystem: "github-actions" + # Workflow files stored in the + # default location of `.github/workflows` + directory: "/" + schedule: + interval: "weekly" + groups: + all-github-actions: + patterns: [ "*" ] + commit-message: + prefix: ":seedling:" + labels: + - "ok-to-test" diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 6029069458..2bbbb33aea 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -4,6 +4,15 @@ on: types: [opened, edited, synchronize, reopened] branches: - main + +permissions: + # Required: allow read access to the content for analysis. + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + pull-requests: read + # Optional: Allow write access to checks to allow the action to annotate code in the PR. + checks: write + jobs: golangci: name: lint @@ -14,13 +23,17 @@ jobs: - "" - tools/setup-envtest steps: - - uses: actions/setup-go@v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + - name: Calculate go version + id: vars + run: echo "go_version=$(make go-version)" >> $GITHUB_OUTPUT + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # tag=v6.0.0 with: - go-version: '1.20' - cache: false - - uses: actions/checkout@v3 + go-version: ${{ steps.vars.outputs.go_version }} - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # tag=v8.0.0 with: - version: v1.53.3 + version: v2.5.0 + args: --output.text.print-linter-name=true --output.text.colors=true --timeout 10m working-directory: ${{matrix.working-directory}} diff --git a/.github/workflows/ossf-scorecard.yaml b/.github/workflows/ossf-scorecard.yaml new file mode 100644 index 0000000000..24156f49e0 --- /dev/null +++ b/.github/workflows/ossf-scorecard.yaml @@ -0,0 +1,56 @@ +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + schedule: + # Weekly on Saturdays. + - cron: '30 1 * * 6' + push: + branches: [ "main" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-latest + permissions: + # Needed if using Code scanning alerts + security-events: write + # Needed for GitHub OIDC token if publish_results is true + id-token: write + + steps: + - name: "Checkout code" + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # tag=v2.4.3 + with: + results_file: results.sarif + results_format: sarif + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + publish_results: true + + # Upload the results as artifacts. + - name: "Upload artifact" + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # tag=v4.6.2 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # required for Code scanning alerts + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@83a02f7883b12e0e4e1a146174f5e2292a01e601 # tag=v2.16.4 + with: + sarif_file: results.sarif diff --git a/.github/workflows/pr-dependabot.yaml b/.github/workflows/pr-dependabot.yaml new file mode 100644 index 0000000000..10162e9129 --- /dev/null +++ b/.github/workflows/pr-dependabot.yaml @@ -0,0 +1,38 @@ +name: PR dependabot go modules fix + +# This action runs on PRs opened by dependabot and updates modules. +on: + pull_request: + branches: + - dependabot/** + push: + branches: + - dependabot/** + workflow_dispatch: + +permissions: + contents: write # Allow to update the PR. + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + - name: Calculate go version + id: vars + run: echo "go_version=$(make go-version)" >> $GITHUB_OUTPUT + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # tag=v6.0.0 + with: + go-version: ${{ steps.vars.outputs.go_version }} + - name: Update all modules + run: make modules + - uses: EndBug/add-and-commit@a94899bca583c204427a224a7af87c02f9b325d5 # tag=v9.1.4 + name: Commit changes + with: + author_name: dependabot[bot] + author_email: 49699333+dependabot[bot]@users.noreply.github.com + default_author: github_actor + message: 'Update generated code' diff --git a/.github/workflows/pr-gh-workflow-approve.yaml b/.github/workflows/pr-gh-workflow-approve.yaml new file mode 100644 index 0000000000..28be4dac71 --- /dev/null +++ b/.github/workflows/pr-gh-workflow-approve.yaml @@ -0,0 +1,42 @@ +name: PR approve GH Workflows + +on: + pull_request_target: + types: + - edited + - labeled + - reopened + - synchronize + +permissions: {} + +jobs: + approve: + name: Approve ok-to-test + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Update PR + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const result = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + event: "pull_request", + status: "action_required", + head_sha: context.payload.pull_request.head.sha, + per_page: 100 + }); + + for (var run of result.data.workflow_runs) { + await github.rest.actions.approveWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id + }); + } diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..6e3b7734e7 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,35 @@ +name: Upload binaries to release + +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + +permissions: + contents: write + +jobs: + build: + name: Upload binaries to release + runs-on: ubuntu-latest + steps: + - name: Set env + run: echo "RELEASE_TAG=${GITHUB_REF:10}" >> $GITHUB_ENV + - name: Check out code + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + - name: Calculate go version + id: vars + run: echo "go_version=$(make go-version)" >> $GITHUB_OUTPUT + - name: Set up Go + uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # tag=v6.0.0 + with: + go-version: ${{ steps.vars.outputs.go_version }} + - name: Generate release binaries + run: | + make release + - name: Release + uses: softprops/action-gh-release@62c96d0c4e8a889135c1f3a25910db8dbe0e85f7 # tag=v2.3.4 + with: + draft: false + files: tools/setup-envtest/out/* diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 51bb083ed5..2168d72516 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1,14 +1,18 @@ +name: PR title verifier + on: pull_request_target: - types: [opened, edited, reopened, synchronize] + types: [opened, edited, synchronize, reopened] jobs: verify: runs-on: ubuntu-latest - name: verify PR contents + steps: - - name: Verifier action - id: verifier - uses: kubernetes-sigs/kubebuilder-release-tools@v0.3.0 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # tag=v5.0.0 + + - name: Check if PR title is valid + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: | + ./hack/verify-pr-title.sh "${PR_TITLE}" diff --git a/.gitignore b/.gitignore index 294685952b..2ddc5a8b87 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,8 @@ # Tools binaries. hack/tools/bin +# Release artifacts +tools/setup-envtest/out + junit-report.xml -/artifacts \ No newline at end of file +/artifacts diff --git a/.golangci.yml b/.golangci.yml index deb6a783da..85701c88a8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,29 +1,33 @@ +version: "2" +run: + go: "1.24" + timeout: 10m + allow-parallel-runners: true linters: - disable-all: true + default: none enable: - asasalint - asciicheck - bidichk - bodyclose + - copyloopvar - dogsled - dupl - errcheck - errchkjson - errorlint - exhaustive - - exportloopref + - forbidigo - ginkgolinter - goconst - gocritic - gocyclo - - gofmt - - goimports + - godoclint - goprintffuncname - - gosec - - gosimple - govet - importas - ineffassign + - iotamixing - makezero - misspell - nakedret @@ -32,139 +36,161 @@ linters: - prealloc - revive - staticcheck - - stylecheck - tagliatelle - - typecheck - unconvert - unparam - unused - whitespace - -linters-settings: - importas: - no-unaliased: true - alias: - # Kubernetes - - pkg: k8s.io/api/core/v1 - alias: corev1 - - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 - alias: apiextensionsv1 - - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 - alias: metav1 - - pkg: k8s.io/apimachinery/pkg/api/errors - alias: apierrors - - pkg: k8s.io/apimachinery/pkg/util/errors - alias: kerrors - # Controller Runtime - - pkg: sigs.k8s.io/controller-runtime - alias: ctrl - staticcheck: - go: "1.20" - stylecheck: - go: "1.20" - revive: + settings: + forbidigo: + forbid: + - pattern: context.Background + msg: Use ginkgos SpecContext or go testings t.Context instead + - pattern: context.TODO + msg: Use ginkgos SpecContext or go testings t.Context instead + govet: + disable: + - fieldalignment + - shadow + enable-all: true + importas: + alias: + - pkg: k8s.io/api/core/v1 + alias: corev1 + - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 + alias: apiextensionsv1 + - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 + alias: metav1 + - pkg: k8s.io/apimachinery/pkg/api/errors + alias: apierrors + - pkg: k8s.io/apimachinery/pkg/util/errors + alias: kerrors + - pkg: sigs.k8s.io/controller-runtime + alias: ctrl + no-unaliased: true + revive: + rules: + # The following rules are recommended https://github.com/mgechev/revive#recommended-configuration + - name: blank-imports + - name: context-as-argument + - name: context-keys-type + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: if-return + - name: increment-decrement + - name: var-naming + - name: var-declaration + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: superfluous-else + - name: unreachable-code + - name: redefines-builtin-id + # + # Rules in addition to the recommended configuration above. + # + - name: bool-literal-in-expr + - name: constant-logical-expr + exclusions: + generated: strict + paths: + - zz_generated.*\.go$ + - .*conversion.*\.go$ rules: - # The following rules are recommended https://github.com/mgechev/revive#recommended-configuration - - name: blank-imports - - name: context-as-argument - - name: context-keys-type - - name: dot-imports - - name: error-return - - name: error-strings - - name: error-naming - - name: exported - - name: if-return - - name: increment-decrement - - name: var-naming - - name: var-declaration - - name: range - - name: receiver-naming - - name: time-naming - - name: unexported-return - - name: indent-error-flow - - name: errorf - - name: superfluous-else - - name: unreachable-code - - name: redefines-builtin-id - # - # Rules in addition to the recommended configuration above. - # - - name: bool-literal-in-expr - - name: constant-logical-expr - + - linters: + - forbidigo + path-except: _test\.go + - linters: + - gosec + text: 'G108: Profiling endpoint is automatically exposed on /debug/pprof' + - linters: + - revive + text: 'exported: exported method .*\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported' + - linters: + - errcheck + text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked + - linters: + - staticcheck + text: 'SA1019: .*The component config package has been deprecated and will be removed in a future release.' + # With Go 1.16, the new embed directive can be used with an un-named import, + # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. + # This directive allows the embed package to be imported with an underscore everywhere. + - linters: + - revive + source: _ "embed" + # Exclude some packages or code to require comments, for example test code, or fake clients. + - linters: + - revive + text: exported (method|function|type|const) (.+) should have comment or be unexported + source: (func|type).*Fake.* + - linters: + - revive + path: fake_\.go + text: exported (method|function|type|const) (.+) should have comment or be unexported + # Disable unparam "always receives" which might not be really + # useful when building libraries. + - linters: + - unparam + text: always receives + # Dot imports for gomega and ginkgo are allowed + # within test files. + - path: _test\.go + text: should not use dot imports + - path: _test\.go + text: cyclomatic complexity + - path: _test\.go + text: 'G107: Potential HTTP request made with variable url' + # Append should be able to assign to a different var/slice. + - linters: + - gocritic + text: 'appendAssign: append result not assigned to the same slice' + - linters: + - gocritic + text: 'singleCaseSwitch: should rewrite switch statement to if statement' + # It considers all file access to a filename that comes from a variable problematic, + # which is naiv at best. + - linters: + - gosec + text: 'G304: Potential file inclusion via variable' + - linters: + - dupl + path: _test\.go + - linters: + - revive + path: .*/internal/.* + - linters: + - unused + # Seems to incorrectly trigger on the two implementations that are only + # used through an interface and not directly..? + # Likely same issue as https://github.com/dominikh/go-tools/issues/1616 + path: pkg/controller/priorityqueue/metrics\.go + # The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time. + # If it is decided they will not be addressed they should be moved above this comment. + - path: (.+)\.go$ + text: Subprocess launch(ed with variable|ing should be audited) + - linters: + - gosec + path: (.+)\.go$ + text: (G204|G104|G307) + - linters: + - staticcheck + path: (.+)\.go$ + text: (ST1000|QF1008) issues: - max-same-issues: 0 max-issues-per-linter: 0 - # We are disabling default golangci exclusions because we want to help reviewers to focus on reviewing the most relevant - # changes in PRs and avoid nitpicking. - exclude-use-default: false - # List of regexps of issue texts to exclude, empty list by default. - exclude: - # The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time. - # If it is decided they will not be addressed they should be moved above this comment. - - Subprocess launch(ed with variable|ing should be audited) - - (G204|G104|G307) - - "ST1000: at least one file in a package should have a package comment" - exclude-rules: - - linters: - - gosec - text: "G108: Profiling endpoint is automatically exposed on /debug/pprof" - - linters: - - revive - text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported" - - linters: - - errcheck - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - - linters: - - staticcheck - text: "SA1019: .*The component config package has been deprecated and will be removed in a future release." - # With Go 1.16, the new embed directive can be used with an un-named import, - # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. - # This directive allows the embed package to be imported with an underscore everywhere. - - linters: - - revive - source: _ "embed" - # Exclude some packages or code to require comments, for example test code, or fake clients. - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - source: (func|type).*Fake.* - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - path: fake_\.go - # Disable unparam "always receives" which might not be really - # useful when building libraries. - - linters: - - unparam - text: always receives - # Dot imports for gomega and ginkgo are allowed - # within test files. - - path: _test\.go - text: should not use dot imports - - path: _test\.go - text: cyclomatic complexity - - path: _test\.go - text: "G107: Potential HTTP request made with variable url" - # Append should be able to assign to a different var/slice. - - linters: - - gocritic - text: "appendAssign: append result not assigned to the same slice" - - linters: - - gocritic - text: "singleCaseSwitch: should rewrite switch statement to if statement" - # It considers all file access to a filename that comes from a variable problematic, - # which is naiv at best. - - linters: - - gosec - text: "G304: Potential file inclusion via variable" - - linters: - - dupl - path: _test\.go - -run: - timeout: 10m - skip-files: - - "zz_generated.*\\.go$" - - ".*conversion.*\\.go$" - allow-parallel-runners: true + max-same-issues: 0 +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: strict + paths: + - zz_generated.*\.go$ + - .*conversion.*\.go$ diff --git a/.gomodcheck.yaml b/.gomodcheck.yaml new file mode 100644 index 0000000000..3608de331d --- /dev/null +++ b/.gomodcheck.yaml @@ -0,0 +1,17 @@ +upstreamRefs: + - k8s.io/api + - k8s.io/apiextensions-apiserver + - k8s.io/apimachinery + - k8s.io/apiserver + - k8s.io/client-go + - k8s.io/component-base + # k8s.io/klog/v2 -> conflicts with k/k deps + # k8s.io/utils -> conflicts with k/k deps + +excludedModules: + # --- test dependencies: + - github.com/onsi/ginkgo/v2 + - github.com/onsi/gomega + + # --- We want a newer version with generics support for this + - github.com/google/btree diff --git a/FAQ.md b/FAQ.md index c21b29e287..9c36c8112e 100644 --- a/FAQ.md +++ b/FAQ.md @@ -4,13 +4,13 @@ **A**: Each controller should only reconcile one object type. Other affected objects should be mapped to a single type of root object, using -the `EnqueueRequestForOwner` or `EnqueueRequestsFromMapFunc` event +the `handler.EnqueueRequestForOwner` or `handler.EnqueueRequestsFromMapFunc` event handlers, and potentially indices. Then, your Reconcile method should attempt to reconcile *all* state for that given root objects. ### Q: How do I have different logic in my reconciler for different types of events (e.g. create, update, delete)? -**A**: You should not. Reconcile functions should be idempotent, and +**A**: You should not. Reconcile functions should be idempotent, and should always reconcile state by reading all the state it needs, then writing updates. This allows your reconciler to correctly respond to generic events, adjust to skipped or coalesced events, and easily deal diff --git a/Makefile b/Makefile index 71ec644de0..b8e9cfa877 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,11 @@ SHELL:=/usr/bin/env bash .DEFAULT_GOAL:=help +# +# Go. +# +GO_VERSION ?= 1.24.0 + # Use GOPROXY environment variable if set GOPROXY := $(shell go env GOPROXY) ifeq ($(GOPROXY),) @@ -34,13 +39,22 @@ export GOPROXY # Active module mode, as we use go modules to manage dependencies export GO111MODULE=on +# Hosts running SELinux need :z added to volume mounts +SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0) + +ifeq ($(SELINUX_ENABLED),1) + DOCKER_VOL_OPTS?=:z +endif + # Tools. TOOLS_DIR := hack/tools -TOOLS_BIN_DIR := $(TOOLS_DIR)/bin +TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/bin) GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint) GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen ENVTEST_DIR := $(abspath tools/setup-envtest) +SCRATCH_ENV_DIR := $(abspath examples/scratch-env) +GO_INSTALL := ./hack/go-install.sh # The help will print out all targets with their descriptions organized bellow their categories. The categories are represented by `##@` and the target descriptions by `##`. # The awk commands is responsible to read the entire set of makefiles included in this invocation, looking for lines of the file as xyz: ## something, and then pretty-format the target and help. Then, if there's a line with ##@ something, that gets pretty-printed as a category. @@ -66,16 +80,36 @@ test-tools: ## tests the tools codebase (setup-envtest) ## Binaries ## -------------------------------------- -$(GO_APIDIFF): $(TOOLS_DIR)/go.mod # Build go-apidiff from tools folder. - cd $(TOOLS_DIR) && go build -tags=tools -o bin/go-apidiff github.com/joelanford/go-apidiff +GO_APIDIFF_VER := v0.8.2 +GO_APIDIFF_BIN := go-apidiff +GO_APIDIFF := $(abspath $(TOOLS_BIN_DIR)/$(GO_APIDIFF_BIN)-$(GO_APIDIFF_VER)) +GO_APIDIFF_PKG := github.com/joelanford/go-apidiff -$(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder. - cd $(TOOLS_DIR) && go build -tags=tools -o bin/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen +$(GO_APIDIFF): # Build go-apidiff from tools folder. + GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GO_APIDIFF_PKG) $(GO_APIDIFF_BIN) $(GO_APIDIFF_VER) -$(GOLANGCI_LINT): .github/workflows/golangci-lint.yml # Download golanci-lint using hack script into tools folder. - hack/ensure-golangci-lint.sh \ - -b $(TOOLS_BIN_DIR) \ - $(shell cat .github/workflows/golangci-lint.yml | grep "version: v" | sed 's/.*version: //') +CONTROLLER_GEN_VER := v0.17.1 +CONTROLLER_GEN_BIN := controller-gen +CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/$(CONTROLLER_GEN_BIN)-$(CONTROLLER_GEN_VER)) +CONTROLLER_GEN_PKG := sigs.k8s.io/controller-tools/cmd/controller-gen + +$(CONTROLLER_GEN): # Build controller-gen from tools folder. + GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(CONTROLLER_GEN_PKG) $(CONTROLLER_GEN_BIN) $(CONTROLLER_GEN_VER) + +GOLANGCI_LINT_BIN := golangci-lint +GOLANGCI_LINT_VER := $(shell cat .github/workflows/golangci-lint.yml | grep [[:space:]]version: | sed 's/.*version: //') +GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/$(GOLANGCI_LINT_BIN)-$(GOLANGCI_LINT_VER)) +GOLANGCI_LINT_PKG := github.com/golangci/golangci-lint/v2/cmd/golangci-lint + +$(GOLANGCI_LINT): # Build golangci-lint from tools folder. + GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOLANGCI_LINT_PKG) $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER) + +GO_MOD_CHECK_DIR := $(abspath ./hack/tools/cmd/gomodcheck) +GO_MOD_CHECK := $(abspath $(TOOLS_BIN_DIR)/gomodcheck) +GO_MOD_CHECK_IGNORE := $(abspath .gomodcheck.yaml) +.PHONY: $(GO_MOD_CHECK) +$(GO_MOD_CHECK): # Build gomodcheck + go build -C $(GO_MOD_CHECK_DIR) -o $(GO_MOD_CHECK) ## -------------------------------------- ## Linting @@ -99,10 +133,49 @@ modules: ## Runs go mod to ensure modules are up to date. go mod tidy cd $(TOOLS_DIR); go mod tidy cd $(ENVTEST_DIR); go mod tidy + cd $(SCRATCH_ENV_DIR); go mod tidy + +## -------------------------------------- +## Release +## -------------------------------------- -.PHONY: generate -generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config file - $(CONTROLLER_GEN) object paths="./pkg/config/v1alpha1/...;./examples/configfile/custom/v1alpha1/..." +RELEASE_DIR := tools/setup-envtest/out + +.PHONY: $(RELEASE_DIR) +$(RELEASE_DIR): + mkdir -p $(RELEASE_DIR)/ + +.PHONY: release +release: clean-release $(RELEASE_DIR) ## Build release. + @if ! [ -z "$$(git status --porcelain)" ]; then echo "Your local git repository contains uncommitted changes, use git clean before proceeding."; exit 1; fi + + # Build binaries first. + $(MAKE) release-binaries + +.PHONY: release-binaries +release-binaries: ## Build release binaries. + RELEASE_BINARY=setup-envtest-linux-amd64 GOOS=linux GOARCH=amd64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-linux-arm64 GOOS=linux GOARCH=arm64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-linux-ppc64le GOOS=linux GOARCH=ppc64le $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-linux-s390x GOOS=linux GOARCH=s390x $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-darwin-amd64 GOOS=darwin GOARCH=amd64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-darwin-arm64 GOOS=darwin GOARCH=arm64 $(MAKE) release-binary + RELEASE_BINARY=setup-envtest-windows-amd64.exe GOOS=windows GOARCH=amd64 $(MAKE) release-binary + +.PHONY: release-binary +release-binary: $(RELEASE_DIR) + docker run \ + --rm \ + -e CGO_ENABLED=0 \ + -e GOOS=$(GOOS) \ + -e GOARCH=$(GOARCH) \ + -e GOCACHE=/tmp/ \ + --user $$(id -u):$$(id -g) \ + -v "$$(pwd):/workspace$(DOCKER_VOL_OPTS)" \ + -w /workspace/tools/setup-envtest \ + golang:$(GO_VERSION) \ + go build -a -trimpath -ldflags "-X 'sigs.k8s.io/controller-runtime/tools/setup-envtest/version.version=$(RELEASE_TAG)' -extldflags '-static'" \ + -o ./out/$(RELEASE_BINARY) ./ ## -------------------------------------- ## Cleanup / Verification @@ -110,22 +183,36 @@ generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config .PHONY: clean clean: ## Cleanup. + $(GOLANGCI_LINT) cache clean $(MAKE) clean-bin .PHONY: clean-bin clean-bin: ## Remove all generated binaries. rm -rf hack/tools/bin +.PHONY: clean-release +clean-release: ## Remove the release folder + rm -rf $(RELEASE_DIR) + .PHONY: verify-modules -verify-modules: modules ## Verify go modules are up to date - @if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(ENVTEST_DIR)/go.mod $(ENVTEST_DIR)/go.sum); then \ +verify-modules: modules $(GO_MOD_CHECK) ## Verify go modules are up to date + @if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(ENVTEST_DIR)/go.mod $(ENVTEST_DIR)/go.sum $(SCRATCH_ENV_DIR)/go.sum); then \ git diff; \ echo "go module files are out of date, please run 'make modules'"; exit 1; \ fi + $(GO_MOD_CHECK) $(GO_MOD_CHECK_IGNORE) -.PHONY: verify-generate -verify-generate: generate ## Verify generated files are up to date - @if !(git diff --quiet HEAD); then \ - git diff; \ - echo "generated files are out of date, run make generate"; exit 1; \ - fi +APIDIFF_OLD_COMMIT ?= $(shell git rev-parse origin/main) + +.PHONY: apidiff +verify-apidiff: $(GO_APIDIFF) ## Check for API differences + $(GO_APIDIFF) $(APIDIFF_OLD_COMMIT) --print-compatible + +## -------------------------------------- +## Helpers +## -------------------------------------- + +##@ helpers: + +go-version: ## Print the go version we use to compile our binaries and images + @echo $(GO_VERSION) diff --git a/OWNERS b/OWNERS index 4b1fa044bf..9f2d296e4c 100644 --- a/OWNERS +++ b/OWNERS @@ -6,5 +6,6 @@ approvers: - controller-runtime-approvers reviewers: - controller-runtime-admins - - controller-runtime-reviewers + - controller-runtime-maintainers - controller-runtime-approvers + - controller-runtime-reviewers diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 7848941d53..47bf6eedf3 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -4,8 +4,10 @@ aliases: # active folks who can be contacted to perform admin-related # tasks on the repo, or otherwise approve any PRS. controller-runtime-admins: - - vincepri + - alvaroaleman - joelanford + - sbueringer + - vincepri # non-admin folks who have write-access and can approve any PRs in the repo controller-runtime-maintainers: @@ -24,12 +26,8 @@ aliases: controller-runtime-reviewers: - varshaprasad96 - inteon - - # folks to can approve things in the directly-ported - # testing_frameworks portions of the codebase - testing-integration-approvers: - - apelisse - - hoegaarden + - JoelSpeed + - troy0820 # folks who may have context on ancient history, # but are no longer directly involved diff --git a/README.md b/README.md index e785abdd77..54bacad42e 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ The full documentation can be found at [VERSIONING.md](VERSIONING.md), but TL;DR Users: -- We follow [Semantic Versioning (semver)](https://semver.org) -- Use releases with your dependency management to ensure that you get compatible code -- The main branch contains all the latest code, some of which may break compatibility (so "normal" `go get` is not recommended) +- We stick to a zero major version +- We publish a minor version for each Kubernetes minor release and allow breaking changes between minor versions +- We publish patch versions as needed and we don't allow breaking changes in them Contributors: @@ -40,6 +40,27 @@ Contributors: * [Documentation Changes](/.github/PULL_REQUEST_TEMPLATE/docs.md) * [Test/Build/Other Changes](/.github/PULL_REQUEST_TEMPLATE/other.md) +## Compatibility + +Every minor version of controller-runtime has been tested with a specific minor version of client-go. A controller-runtime minor version *may* be compatible with +other client-go minor versions, but this is by chance and neither supported nor tested. In general, we create one minor version of controller-runtime +for each minor version of client-go and other k8s.io/* dependencies. + +The minimum Go version of controller-runtime is the highest minimum Go version of our Go dependencies. Usually, this will +be identical to the minimum Go version of the corresponding k8s.io/* dependencies. + +Compatible k8s.io/*, client-go and minimum Go versions can be looked up in our [go.mod](go.mod) file. + +| | k8s.io/*, client-go | minimum Go version | +|----------|:-------------------:|:------------------:| +| CR v0.21 | v0.33 | 1.24 | +| CR v0.20 | v0.32 | 1.23 | +| CR v0.19 | v0.31 | 1.22 | +| CR v0.18 | v0.30 | 1.22 | +| CR v0.17 | v0.29 | 1.21 | +| CR v0.16 | v0.28 | 1.20 | +| CR v0.15 | v0.27 | 1.20 | + ## FAQ See [FAQ.md](FAQ.md) @@ -48,15 +69,13 @@ See [FAQ.md](FAQ.md) Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/). -controller-runtime is a subproject of the [kubebuilder](https://github.com/kubernetes-sigs/kubebuilder) project -in sig apimachinery. - You can reach the maintainers of this project at: - Slack channel: [#controller-runtime](https://kubernetes.slack.com/archives/C02MRBMN00Z) - Google Group: [kubebuilder@googlegroups.com](https://groups.google.com/forum/#!forum/kubebuilder) ## Contributing + Contributions are greatly appreciated. The maintainers actively manage the issues list, and try to highlight issues suitable for newcomers. The project follows the typical GitHub pull request model. See [CONTRIBUTING.md](CONTRIBUTING.md) for more details. Before starting any work, please either comment on an existing issue, or file a new one. diff --git a/RELEASE.md b/RELEASE.md index f234494fe1..2a857b976e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -4,9 +4,9 @@ The Kubernetes controller-runtime Project is released on an as-needed basis. The **Note:** Releases are done from the `release-MAJOR.MINOR` branches. For PATCH releases is not required to create a new branch you will just need to ensure that all big fixes are cherry-picked into the respective -`release-MAJOR.MINOR` branch. To know more about versioning check https://semver.org/. +`release-MAJOR.MINOR` branch. To know more about versioning check https://semver.org/. -## How to do a release +## How to do a release ### Create the new branch and the release tag @@ -15,7 +15,7 @@ to create a new branch you will just need to ensure that all big fixes are cherr ### Now, let's generate the changelog -1. Create the changelog from the new branch `release-` (`git checkout release-`). +1. Create the changelog from the new branch `release-` (`git checkout release-`). You will need to use the [kubebuilder-release-tools][kubebuilder-release-tools] to generate the notes. See [here][release-notes-generation] > **Note** @@ -24,12 +24,12 @@ You will need to use the [kubebuilder-release-tools][kubebuilder-release-tools] ### Draft a new release from GitHub -1. Create a new tag with the correct version from the new `release-` branch +1. Create a new tag with the correct version from the new `release-` branch 2. Add the changelog on it and publish. Now, the code source is released ! ### Add a new Prow test the for the new branch release -1. Create a new prow test under [github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime](https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime) +1. Create a new prow test under [github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime](https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime) for the new `release-` branch. (i.e. for the `0.11.0` release see the PR: https://github.com/kubernetes/test-infra/pull/25205) 2. Ping the infra PR in the controller-runtime slack channel for reviews. @@ -45,3 +45,7 @@ For more info, see the release page: https://github.com/kubernetes-sigs/controll ```` 2. An announcement email is sent to `kubebuilder@googlegroups.com` with the subject `[ANNOUNCE] Controller-Runtime $VERSION is released` + +[kubebuilder-release-tools]: https://github.com/kubernetes-sigs/kubebuilder-release-tools +[release-notes-generation]: https://github.com/kubernetes-sigs/kubebuilder-release-tools/blob/master/README.md#release-notes-generation +[release-process]: https://github.com/kubernetes-sigs/kubebuilder/blob/master/VERSIONING.md#releasing diff --git a/VERSIONING.md b/VERSIONING.md index 2c0f2f9b2d..7ad6b142cc 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -7,6 +7,16 @@ For the purposes of the aforementioned guidelines, controller-runtime counts as a "library project", but otherwise follows the guidelines exactly. +We stick to a major version of zero and create a minor version for +each Kubernetes minor version and we allow breaking changes in our +minor versions. We create patch releases as needed and don't allow +breaking changes in them. + +Publishing a non-zero major version is pointless for us, as the k8s.io/* +libraries we heavily depend on do breaking changes but use the same +versioning scheme as described above. Consequently, a project can only +ever depend on one controller-runtime version. + [guidelines]: https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md ## Compatibility and Release Support diff --git a/alias.go b/alias.go index 237963889c..01ba012dcc 100644 --- a/alias.go +++ b/alias.go @@ -21,7 +21,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client/config" - cfg "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -78,6 +77,9 @@ var ( // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // + // The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and + // fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. + // // Will log an error and exit if there is an error creating the rest.Config. GetConfigOrDie = config.GetConfigOrDie @@ -85,6 +87,9 @@ var ( // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // + // The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and + // fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. + // // Config precedence // // * --kubeconfig flag pointing at a file @@ -96,13 +101,6 @@ var ( // * $HOME/.kube/config if exists. GetConfig = config.GetConfig - // ConfigFile returns the cfg.File function for deferred config file loading, - // this is passed into Options{}.From() to populate the Options fields for - // the manager. - // - // Deprecated: This is deprecated in favor of using Options directly. - ConfigFile = cfg.File - // NewControllerManagedBy returns a new controller builder that will be started by the provided Manager. NewControllerManagedBy = builder.ControllerManagedBy @@ -110,6 +108,9 @@ var ( NewWebhookManagedBy = builder.WebhookManagedBy // NewManager returns a new Manager for creating Controllers. + // Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf" + // will be used for all built-in resources of Kubernetes, and "application/json" is for other types + // including all CRD resources. NewManager = manager.New // CreateOrUpdate creates or updates the given object obj in the Kubernetes @@ -127,8 +128,8 @@ var ( // there is another OwnerReference with Controller flag set. SetControllerReference = controllerutil.SetControllerReference - // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned - // which is closed on one of these signals. If a second signal is caught, the program + // SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned + // which is canceled on one of these signals. If a second signal is caught, the program // is terminated with exit code 1. SetupSignalHandler = signals.SetupSignalHandler diff --git a/designs/cache_options.md b/designs/cache_options.md index 94dadffaeb..bdd29c0481 100644 --- a/designs/cache_options.md +++ b/designs/cache_options.md @@ -66,7 +66,7 @@ type ByObject struct { // An empty map prevents this. // // This must be unset for cluster-scoped objects. - Namespaces map[string]*Config + Namespaces map[string]Config // Config will be used for cluster-scoped objects and to default // Config in the Namespaces field. @@ -79,7 +79,7 @@ type ByObject struct { type Options struct { // ByObject specifies per-object cache settings. If unset for a given // object, this will fall through to Default* settings. - ByObject map[client.Object]*ByObject + ByObject map[client.Object]ByObject // DefaultNamespaces maps namespace names to cache settings. If set, it // will be used for all objects that have a nil Namespaces setting. @@ -91,7 +91,7 @@ type Options struct { // // The options in the Config that are nil will be defaulted from // the respective Default* settings. - DefaultNamespaces map[string]*Config + DefaultNamespaces map[string]Config // DefaultLabelSelector is the label selector that will be used as // the default field label selector for everything that doesn't @@ -158,14 +158,14 @@ type Options struct { ``` cache.Options{ - ByObject: map[client.Object]*cache.ByObject{ + ByObject: map[client.Object]cache.ByObject{ &corev1.ConfigMap{}: { - Namespaces: map[string]*cache.Config{ + Namespaces: map[string]cache.Config{ "public": {}, "kube-system": {}, }, }, - &corev1.Secret{}: {Namespaces: map[string]*Config{ + &corev1.Secret{}: {Namespaces: map[string]Config{ "operator": {}, }}, }, @@ -176,9 +176,9 @@ cache.Options{ ``` cache.Options{ - ByObject: map[client.Object]*cache.ByObject{ + ByObject: map[client.Object]cache.ByObject{ &corev1.ConfigMap{}: { - Namespaces: map[string]*cache.Config{ + Namespaces: map[string]cache.Config{ cache.AllNamespaces: nil, // No selector for all namespaces... "operator": {LabelSelector: labelSelector}, // except for the operator namespace }, @@ -192,13 +192,13 @@ cache.Options{ ``` cache.Options{ - ByObject: map[client.Object]*cache.ByObject{ - &appsv1.Deployment: {Namespaces: map[string]*cache.Config{ - cache.AllNamespaces: nil, + ByObject: map[client.Object]cache.ByObject{ + &appsv1.Deployment: {Namespaces: map[string]cache.Config{ + cache.AllNamespaces: {}}, }}, }, - DefaultNamespaces: map[string]*cache.Config{ - "operator": nil, + DefaultNamespaces: map[string]cache.Config{ + "operator": {}}, }, } ``` @@ -207,7 +207,7 @@ cache.Options{ ``` cache.Options{ - ByObject: map[client.Object]*cache.ByObject{ + ByObject: map[client.Object]cache.ByObject{ &corev1.Node: {LabelSelector: labels.Everything()}, }, DefaultLabelSelector: myLabelSelector, @@ -218,9 +218,9 @@ cache.Options{ ``` cache.Options{ - DefaultNamespaces: map[string]*cache.Config{ - "foo": nil, - "bar": nil, + DefaultNamespaces: map[string]cache.Config{ + "foo": {}, + "bar": {}, } } ``` diff --git a/designs/priorityqueue.md b/designs/priorityqueue.md new file mode 100644 index 0000000000..ef1f7588a6 --- /dev/null +++ b/designs/priorityqueue.md @@ -0,0 +1,110 @@ +Priority Queue +=================== + +This document describes the motivation behind implementing a priority queue +in controller-runtime and its design details. + +## Motivation + +1. Controllers reconcile all objects during startup to account for changes in + the reconciliation logic. Some controllers also periodically re-reconcile + everything to account for out of band changes they do not get notified for, + this is for example common for controllers managing cloud resources. In both + these cases, the reconciliation of new or changed objects gets delayed, + resulting in poor user experience. [Example][0] +2. There may be application-specific reason why some events are more important + than others, [Example][1] + +## Proposed changes + +Implement a priority queue in controller-runtime that exposes the following +interface: + +```go +type PriorityQueue[T comparable] interface { + // AddWithOpts adds one or more items to the workqueue. Items + // in the workqueue are de-duplicated, so there will only ever + // be one entry for a given key. + // Adding an item that is already there may update its wait + // period to the lowest of existing and new wait period or + // its priority to the highest of existing and new priority. + AddWithOpts(o AddOpts, items ...T) + + // GetWithPriority returns an item and its priority. It allows + // a controller to re-use the priority if it enqueues an item + // again. + GetWithPriority() (item T, priority int, shutdown bool) + + // workqueue.TypedRateLimitingInterface is kept for backwards + // compatibility. + workqueue.TypedRateLimitingInterface[T] +} + +type AddOpts struct { + // After is a duration after which the object will be available for + // reconciliation. If the object is already in the workqueue, the + // lowest of existing and new After period will be used. + After time.Duration + + // Ratelimited specifies if the ratelimiter should be used to + // determine a wait period. If the object is already in the + // workqueue, the lowest of existing and new wait period will be + // used. + RateLimited bool + + // Priority specifies the priority of the object. Objects with higher + // priority are returned before objects with lower priority. If the + // object is already in the workqueue, the priority will be updated + // to the highest of existing and new priority. + // + // The default value is 0. + Priority int +} +``` + +In order to fix the issue described in point one of the motivation section, +we have to be able to differentiate events stemming from the initial list +during startup and from resyncs from other events. For events from the initial +list, the informer emits a `Create` event whereas for `Resync` it emits an `Update` +event. The suggestion is to use a heuristic for `Create` events, if the object +in there is older than one minute, it is assumed to be from the initial `List`. +For the `Resync`, we simply check if the `ResourceVersion` is unchanged. +In both these cases, we will lower the priority to `LowPriority`/`-100`. +This gives some room for use-cases where people want to use a priority that +is lower than default (`0`) but higher than what we use in the wrapper. + +```go +// WithLowPriorityWhenUnchanged wraps an existing handler and will +// reduce the priority of events stemming from the initial listwatch +// or cache resyncs to LowPriority. +func WithLowPriorityWhenUnchanged[object client.Object, request comparable](u TypedEventHandler[object, request]) TypedEventHandler[object, request]{ +} +``` + +```go +// LowPriority is the priority set by WithLowPriorityWhenUnchanged +const LowPriority = -100 +``` + +The issue described in point two of the motivation section ("application-specific +reasons to prioritize some events") will always require implementation of a custom +handler or eventsource in order to inject the appropriate priority. + +## Implementation stages + +In order to safely roll this out to all controller-runtime users, it is suggested to +divide the implementation into two stages: Initially, we will add the priority queue +but mark it as experimental and all usage of it requires explicit opt-in by setting +a boolean on the manager or configuring `NewQueue` in a controllers opts. There will +be no breaking changes required for this, but sources or handlers that want to make +use of the new queue will have to use type assertions. + +After we've gained some confidence that the implementation is useful and correct, we +will make it the default. Doing so entails breaking the `source.Source` and the +`handler.Handler` interfaces as well as the `controller.Options` struct to refer to +the new workqueue interface. We will wait at least one minor release after introducing +the `PriorityQueue` before doing this. + + +* [0]: https://youtu.be/AYNaaXlV8LQ?si=i2Pfo7Ske6rTrPLS +* [1]: https://github.com/cilium/cilium/blob/a17d6945b29c177209af3d985bd82cce49eed4a1/operator/pkg/ciliumendpointslice/controller.go#L73 diff --git a/doc.go b/doc.go index 0319bc3ff8..75d1d908c5 100644 --- a/doc.go +++ b/doc.go @@ -87,7 +87,7 @@ limitations under the License. // during writes (nor does it promise sequential create/get coherence), and code // should not assume a get immediately following a create/update will return // the updated resource. Caches may also have indexes, which can be created via -// a FieldIndexer (pkg/client) obtained from the manager. Indexes can used to +// a FieldIndexer (pkg/client) obtained from the manager. Indexes can be used to // quickly and easily look up all objects with certain fields set. Reconcilers // may retrieve event recorders (pkg/recorder) to emit events using the // manager. diff --git a/example_test.go b/example_test.go index 381959d5bb..cbbf032b0f 100644 --- a/example_test.go +++ b/example_test.go @@ -18,14 +18,21 @@ package controllerruntime_test import ( "context" + "encoding/json" "fmt" "os" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" // since we invoke tests with -ginkgo.junit-report we need to import ginkgo. _ "github.com/onsi/ginkgo/v2" @@ -38,7 +45,7 @@ import ( // // * Start the application. func Example() { - var log = ctrl.Log.WithName("builder-examples") + log := ctrl.Log.WithName("builder-examples") manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) if err != nil { @@ -62,6 +69,94 @@ func Example() { } } +type ExampleCRDWithConfigMapRef struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + ConfigMapRef corev1.LocalObjectReference `json:"configMapRef"` +} + +func deepCopyObject(arg any) runtime.Object { + // DO NOT use this code in production code, this is only for presentation purposes. + // in real code you should generate DeepCopy methods by using controller-gen CLI tool. + argBytes, err := json.Marshal(arg) + if err != nil { + panic(err) + } + out := &ExampleCRDWithConfigMapRefList{} + if err := json.Unmarshal(argBytes, out); err != nil { + panic(err) + } + return out +} + +// DeepCopyObject implements client.Object. +func (in *ExampleCRDWithConfigMapRef) DeepCopyObject() runtime.Object { + return deepCopyObject(in) +} + +type ExampleCRDWithConfigMapRefList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ExampleCRDWithConfigMapRef `json:"items"` +} + +// DeepCopyObject implements client.ObjectList. +func (in *ExampleCRDWithConfigMapRefList) DeepCopyObject() runtime.Object { + return deepCopyObject(in) +} + +// This example creates a simple application Controller that is configured for ExampleCRDWithConfigMapRef CRD. +// Any change in the configMap referenced in this Custom Resource will cause the re-reconcile of the parent ExampleCRDWithConfigMapRef +// due to the implementation of the .Watches method of "sigs.k8s.io/controller-runtime/pkg/builder".Builder. +func Example_customHandler() { + log := ctrl.Log.WithName("builder-examples") + + manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + if err != nil { + log.Error(err, "could not create manager") + os.Exit(1) + } + + err = ctrl. + NewControllerManagedBy(manager). + For(&ExampleCRDWithConfigMapRef{}). + Watches(&corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, cm client.Object) []ctrl.Request { + // map a change from referenced configMap to ExampleCRDWithConfigMapRef, which causes its re-reconcile + crList := &ExampleCRDWithConfigMapRefList{} + if err := manager.GetClient().List(ctx, crList); err != nil { + manager.GetLogger().Error(err, "while listing ExampleCRDWithConfigMapRefs") + return nil + } + + reqs := make([]ctrl.Request, 0, len(crList.Items)) + for _, item := range crList.Items { + if item.ConfigMapRef.Name == cm.GetName() { + reqs = append(reqs, ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: item.GetNamespace(), + Name: item.GetName(), + }, + }) + } + } + + return reqs + })). + Complete(reconcile.Func(func(ctx context.Context, r reconcile.Request) (reconcile.Result, error) { + // Your business logic to implement the API by creating, updating, deleting objects goes here. + return reconcile.Result{}, nil + })) + if err != nil { + log.Error(err, "could not create controller") + os.Exit(1) + } + + if err := manager.Start(ctrl.SetupSignalHandler()); err != nil { + log.Error(err, "could not start manager") + os.Exit(1) + } +} + // This example creates a simple application Controller that is configured for ReplicaSets and Pods. // This application controller will be running leader election with the provided configuration in the manager options. // If leader election configuration is not provided, controller runs leader election with default values. @@ -75,7 +170,7 @@ func Example() { // // * Start the application. func Example_updateLeaderElectionDurations() { - var log = ctrl.Log.WithName("builder-examples") + log := ctrl.Log.WithName("builder-examples") leaseDuration := 100 * time.Second renewDeadline := 80 * time.Second retryPeriod := 20 * time.Second diff --git a/examples/builtins/controller.go b/examples/builtins/controller.go index 6c8c5d935f..443283140a 100644 --- a/examples/builtins/controller.go +++ b/examples/builtins/controller.go @@ -21,7 +21,7 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -43,13 +43,13 @@ func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.R // Fetch the ReplicaSet from the cache rs := &appsv1.ReplicaSet{} err := r.client.Get(ctx, request.NamespacedName, rs) - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { log.Error(nil, "Could not find ReplicaSet") return reconcile.Result{}, nil } if err != nil { - return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+v", err) + return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+w", err) } // Print the ReplicaSet @@ -67,7 +67,7 @@ func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.R rs.Labels["hello"] = "world" err = r.client.Update(ctx, rs) if err != nil { - return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+v", err) + return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+w", err) } return reconcile.Result{}, nil diff --git a/examples/builtins/main.go b/examples/builtins/main.go index 8ea173b248..3a47814d8c 100644 --- a/examples/builtins/main.go +++ b/examples/builtins/main.go @@ -22,23 +22,19 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - "sigs.k8s.io/controller-runtime/pkg/builder" + + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/source" ) -func init() { - log.SetLogger(zap.New()) -} - func main() { - entryLog := log.Log.WithName("entrypoint") + ctrl.SetLogger(zap.New()) + entryLog := ctrl.Log.WithName("entrypoint") // Setup a Manager entryLog.Info("setting up manager") @@ -50,28 +46,21 @@ func main() { // Setup a new controller to reconcile ReplicaSets entryLog.Info("Setting up controller") - c, err := controller.New("foo-controller", mgr, controller.Options{ - Reconciler: &reconcileReplicaSet{client: mgr.GetClient()}, - }) - if err != nil { - entryLog.Error(err, "unable to set up individual controller") - os.Exit(1) - } - // Watch ReplicaSets and enqueue ReplicaSet object key - if err := c.Watch(source.Kind(mgr.GetCache(), &appsv1.ReplicaSet{}), &handler.EnqueueRequestForObject{}); err != nil { - entryLog.Error(err, "unable to watch ReplicaSets") - os.Exit(1) - } - - // Watch Pods and enqueue owning ReplicaSet key - if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), - handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.ReplicaSet{}, handler.OnlyControllerOwner())); err != nil { - entryLog.Error(err, "unable to watch Pods") + err = ctrl. + NewControllerManagedBy(mgr). + Named("foo-controller"). + WatchesRawSource(source.Kind(mgr.GetCache(), &appsv1.ReplicaSet{}, + &handler.TypedEnqueueRequestForObject[*appsv1.ReplicaSet]{})). + WatchesRawSource(source.Kind(mgr.GetCache(), &corev1.Pod{}, + handler.TypedEnqueueRequestForOwner[*corev1.Pod](mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()))). + Complete(&reconcileReplicaSet{client: mgr.GetClient()}) + if err != nil { + entryLog.Error(err, "could not create controller") os.Exit(1) } - if err := builder.WebhookManagedBy(mgr). + if err := ctrl.NewWebhookManagedBy(mgr). For(&corev1.Pod{}). WithDefaulter(&podAnnotator{}). WithValidator(&podValidator{}). diff --git a/examples/configfile/builtin/config.yaml b/examples/configfile/builtin/config.yaml deleted file mode 100644 index 39ac86ce60..0000000000 --- a/examples/configfile/builtin/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfiguration -cacheNamespace: default -metrics: - bindAddress: :9091 -leaderElection: - leaderElect: false diff --git a/examples/configfile/builtin/controller.go b/examples/configfile/builtin/controller.go deleted file mode 100644 index 8349bcd5aa..0000000000 --- a/examples/configfile/builtin/controller.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "fmt" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -// reconcileReplicaSet reconciles ReplicaSets -type reconcileReplicaSet struct { - // client can be used to retrieve objects from the APIServer. - client client.Client -} - -// Implement reconcile.Reconciler so the controller can reconcile objects -var _ reconcile.Reconciler = &reconcileReplicaSet{} - -func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // set up a convenient log object so we don't have to type request over and over again - log := log.FromContext(ctx) - - // Fetch the ReplicaSet from the cache - rs := &appsv1.ReplicaSet{} - err := r.client.Get(context.TODO(), request.NamespacedName, rs) - if errors.IsNotFound(err) { - log.Error(nil, "Could not find ReplicaSet") - return reconcile.Result{}, nil - } - - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+v", err) - } - - // Print the ReplicaSet - log.Info("Reconciling ReplicaSet", "container name", rs.Spec.Template.Spec.Containers[0].Name) - - // Set the label if it is missing - if rs.Labels == nil { - rs.Labels = map[string]string{} - } - if rs.Labels["hello"] == "world" { - return reconcile.Result{}, nil - } - - // Update the ReplicaSet - rs.Labels["hello"] = "world" - err = r.client.Update(context.TODO(), rs) - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+v", err) - } - - return reconcile.Result{}, nil -} diff --git a/examples/configfile/builtin/main.go b/examples/configfile/builtin/main.go deleted file mode 100644 index abd6180d19..0000000000 --- a/examples/configfile/builtin/main.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client/config" - cfg "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager/signals" -) - -var scheme = runtime.NewScheme() - -func init() { - log.SetLogger(zap.New()) - clientgoscheme.AddToScheme(scheme) -} - -func main() { - entryLog := log.Log.WithName("entrypoint") - - // Setup a Manager - entryLog.Info("setting up manager") - mgr, err := ctrl.NewManager(config.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - }.AndFromOrDie(cfg.File())) - if err != nil { - entryLog.Error(err, "unable to set up overall controller manager") - os.Exit(1) - } - - // Setup a new controller to reconcile ReplicaSets - err = ctrl.NewControllerManagedBy(mgr). - For(&appsv1.ReplicaSet{}). - Owns(&corev1.Pod{}). - Complete(&reconcileReplicaSet{ - client: mgr.GetClient(), - }) - if err != nil { - entryLog.Error(err, "unable to create controller") - os.Exit(1) - } - - entryLog.Info("starting manager") - if err := mgr.Start(signals.SetupSignalHandler()); err != nil { - entryLog.Error(err, "unable to run manager") - os.Exit(1) - } -} diff --git a/examples/configfile/custom/config.yaml b/examples/configfile/custom/config.yaml deleted file mode 100644 index bf9ac044b4..0000000000 --- a/examples/configfile/custom/config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: examples.x-k8s.io/v1alpha1 -kind: CustomControllerManagerConfiguration -clusterName: example-test -cacheNamespace: default -metrics: - bindAddress: :8081 -leaderElection: - leaderElect: false diff --git a/examples/configfile/custom/controller.go b/examples/configfile/custom/controller.go deleted file mode 100644 index 8349bcd5aa..0000000000 --- a/examples/configfile/custom/controller.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "context" - "fmt" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -// reconcileReplicaSet reconciles ReplicaSets -type reconcileReplicaSet struct { - // client can be used to retrieve objects from the APIServer. - client client.Client -} - -// Implement reconcile.Reconciler so the controller can reconcile objects -var _ reconcile.Reconciler = &reconcileReplicaSet{} - -func (r *reconcileReplicaSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - // set up a convenient log object so we don't have to type request over and over again - log := log.FromContext(ctx) - - // Fetch the ReplicaSet from the cache - rs := &appsv1.ReplicaSet{} - err := r.client.Get(context.TODO(), request.NamespacedName, rs) - if errors.IsNotFound(err) { - log.Error(nil, "Could not find ReplicaSet") - return reconcile.Result{}, nil - } - - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not fetch ReplicaSet: %+v", err) - } - - // Print the ReplicaSet - log.Info("Reconciling ReplicaSet", "container name", rs.Spec.Template.Spec.Containers[0].Name) - - // Set the label if it is missing - if rs.Labels == nil { - rs.Labels = map[string]string{} - } - if rs.Labels["hello"] == "world" { - return reconcile.Result{}, nil - } - - // Update the ReplicaSet - rs.Labels["hello"] = "world" - err = r.client.Update(context.TODO(), rs) - if err != nil { - return reconcile.Result{}, fmt.Errorf("could not write ReplicaSet: %+v", err) - } - - return reconcile.Result{}, nil -} diff --git a/examples/configfile/custom/main.go b/examples/configfile/custom/main.go deleted file mode 100644 index e0fc95e337..0000000000 --- a/examples/configfile/custom/main.go +++ /dev/null @@ -1,78 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "os" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/examples/configfile/custom/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/client/config" - cfg "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/manager/signals" -) - -var scheme = runtime.NewScheme() - -func init() { - log.SetLogger(zap.New()) - clientgoscheme.AddToScheme(scheme) - v1alpha1.AddToScheme(scheme) -} - -func main() { - entryLog := log.Log.WithName("entrypoint") - - // Setup a Manager - entryLog.Info("setting up manager") - ctrlConfig := v1alpha1.CustomControllerManagerConfiguration{} - - mgr, err := ctrl.NewManager(config.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - }.AndFromOrDie(cfg.File().OfKind(&ctrlConfig))) - if err != nil { - entryLog.Error(err, "unable to set up overall controller manager") - os.Exit(1) - } - - entryLog.Info("setting up cluster", "name", ctrlConfig.ClusterName) - - // Watch ReplicaSets and enqueue ReplicaSet object key - err = ctrl.NewControllerManagedBy(mgr). - For(&appsv1.ReplicaSet{}). - Owns(&corev1.Pod{}). - Complete(&reconcileReplicaSet{ - client: mgr.GetClient(), - }) - if err != nil { - entryLog.Error(err, "unable to create controller") - os.Exit(1) - } - - entryLog.Info("starting manager") - if err := mgr.Start(signals.SetupSignalHandler()); err != nil { - entryLog.Error(err, "unable to run manager") - os.Exit(1) - } -} diff --git a/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go b/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index b9d3b6b4b9..0000000000 --- a/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,35 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CustomControllerManagerConfiguration) DeepCopyInto(out *CustomControllerManagerConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomControllerManagerConfiguration. -func (in *CustomControllerManagerConfiguration) DeepCopy() *CustomControllerManagerConfiguration { - if in == nil { - return nil - } - out := new(CustomControllerManagerConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *CustomControllerManagerConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} diff --git a/examples/crd/main.go b/examples/crd/main.go index 1f6cd5fac2..0bf65c9890 100644 --- a/examples/crd/main.go +++ b/examples/crd/main.go @@ -65,7 +65,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu if podFound { shouldStop := chaospod.Spec.NextStop.Time.Before(time.Now()) if !shouldStop { - return ctrl.Result{RequeueAfter: chaospod.Spec.NextStop.Sub(time.Now()) + 1*time.Second}, nil + return ctrl.Result{RequeueAfter: time.Until(chaospod.Spec.NextStop.Time) + 1*time.Second}, nil } if err := r.Delete(ctx, &pod); err != nil { diff --git a/examples/crd/pkg/groupversion_info.go b/examples/crd/pkg/groupversion_info.go index 04953dd939..693d255b05 100644 --- a/examples/crd/pkg/groupversion_info.go +++ b/examples/crd/pkg/groupversion_info.go @@ -14,19 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package pkg contains API Schema definitions for the chaosapps v1 API group // +kubebuilder:object:generate=true // +groupName=chaosapps.metamagical.io package pkg import ( "k8s.io/apimachinery/pkg/runtime/schema" - logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( - log = logf.Log.WithName("chaospod-resource") - // SchemeGroupVersion is group version used to register these objects SchemeGroupVersion = schema.GroupVersion{Group: "chaosapps.metamagical.io", Version: "v1"} diff --git a/examples/crd/pkg/resource.go b/examples/crd/pkg/resource.go index 555029f5de..80800a23cb 100644 --- a/examples/crd/pkg/resource.go +++ b/examples/crd/pkg/resource.go @@ -17,14 +17,8 @@ limitations under the License. package pkg import ( - "fmt" - "time" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/webhook" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) // ChaosPodSpec defines the desired state of ChaosPod @@ -62,61 +56,6 @@ type ChaosPodList struct { Items []ChaosPod `json:"items"` } -// +kubebuilder:webhook:path=/validate-chaosapps-metamagical-io-v1-chaospod,mutating=false,failurePolicy=fail,groups=chaosapps.metamagical.io,resources=chaospods,verbs=create;update,versions=v1,name=vchaospod.kb.io - -var _ webhook.Validator = &ChaosPod{} - -// ValidateCreate implements webhookutil.validator so a webhook will be registered for the type -func (c *ChaosPod) ValidateCreate() (admission.Warnings, error) { - log.Info("validate create", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - return nil, fmt.Errorf(".spec.nextStop must be later than current time") - } - return nil, nil -} - -// ValidateUpdate implements webhookutil.validator so a webhook will be registered for the type -func (c *ChaosPod) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - log.Info("validate update", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - return nil, fmt.Errorf(".spec.nextStop must be later than current time") - } - - oldC, ok := old.(*ChaosPod) - if !ok { - return nil, fmt.Errorf("expect old object to be a %T instead of %T", oldC, old) - } - if c.Spec.NextStop.After(oldC.Spec.NextStop.Add(time.Hour)) { - return nil, fmt.Errorf("it is not allowed to delay.spec.nextStop for more than 1 hour") - } - return nil, nil -} - -// ValidateDelete implements webhookutil.validator so a webhook will be registered for the type -func (c *ChaosPod) ValidateDelete() (admission.Warnings, error) { - log.Info("validate delete", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - return nil, fmt.Errorf(".spec.nextStop must be later than current time") - } - return nil, nil -} - -// +kubebuilder:webhook:path=/mutate-chaosapps-metamagical-io-v1-chaospod,mutating=true,failurePolicy=fail,groups=chaosapps.metamagical.io,resources=chaospods,verbs=create;update,versions=v1,name=mchaospod.kb.io - -var _ webhook.Defaulter = &ChaosPod{} - -// Default implements webhookutil.defaulter so a webhook will be registered for the type -func (c *ChaosPod) Default() { - log.Info("default", "name", c.Name) - - if c.Spec.NextStop.Before(&metav1.Time{Time: time.Now()}) { - c.Spec.NextStop = metav1.Time{Time: time.Now().Add(time.Minute)} - } -} - func init() { SchemeBuilder.Register(&ChaosPod{}, &ChaosPodList{}) } diff --git a/examples/multiclustersync/main.go b/examples/multiclustersync/main.go new file mode 100644 index 0000000000..e06b754222 --- /dev/null +++ b/examples/multiclustersync/main.go @@ -0,0 +1,178 @@ +package main + +import ( + "context" + "fmt" + "os" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +const ( + sourceNamespace = "namespace-to-sync-all-secrets-from" + targetNamespace = "namespace-to-sync-all-secrets-to" +) + +func run() error { + log.SetLogger(zap.New()) + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + if err != nil { + return fmt.Errorf("failed to construct manager: %w", err) + } + + allTargets := map[string]cluster.Cluster{} + + cluster, err := cluster.New(ctrl.GetConfigOrDie()) + if err != nil { + return fmt.Errorf("failed to construct clusters: %w", err) + } + if err := mgr.Add(cluster); err != nil { + return fmt.Errorf("failed to add cluster to manager: %w", err) + } + + // Add more target clusters here as needed + allTargets["self"] = cluster + + b := builder.TypedControllerManagedBy[request](mgr). + Named("secret-sync"). + // Watch secrets in the source namespace of the source cluster and + // create requests for each target cluster + WatchesRawSource(source.TypedKind( + mgr.GetCache(), + &corev1.Secret{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, s *corev1.Secret) []request { + if s.Namespace != sourceNamespace { + return nil + } + + result := make([]request, 0, len(allTargets)) + for targetCluster := range allTargets { + result = append(result, request{ + NamespacedName: types.NamespacedName{Namespace: s.Namespace, Name: s.Name}, + clusterName: targetCluster, + }) + } + + return result + }), + )). + WithOptions(controller.TypedOptions[request]{MaxConcurrentReconciles: 10}) + + for targetClusterName, targetCluster := range allTargets { + // Watch secrets in the target namespace of each target cluster + // and create a request for itself. + b = b.WatchesRawSource(source.TypedKind( + targetCluster.GetCache(), + &corev1.Secret{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, s *corev1.Secret) []request { + if s.Namespace != targetNamespace { + return nil + } + + return []request{{ + NamespacedName: types.NamespacedName{Namespace: sourceNamespace, Name: s.Name}, + clusterName: targetClusterName, + }} + }), + )) + } + + clients := make(map[string]client.Client, len(allTargets)) + for targetClusterName, targetCluster := range allTargets { + clients[targetClusterName] = targetCluster.GetClient() + } + + if err := b.Complete(&secretSyncReconciler{ + source: mgr.GetClient(), + targets: clients, + }); err != nil { + return fmt.Errorf("failed to build reconciler: %w", err) + } + + ctx := signals.SetupSignalHandler() + if err := mgr.Start(ctx); err != nil { + return fmt.Errorf("failed to start manager: %w", err) + } + + return nil +} + +type request struct { + types.NamespacedName + clusterName string +} + +// secretSyncReconciler is a simple reconciler that keeps all secrets in the source namespace of a given +// source cluster in sync with the secrets in the target namespace of all target clusters. +type secretSyncReconciler struct { + source client.Client + targets map[string]client.Client +} + +func (s *secretSyncReconciler) Reconcile(ctx context.Context, req request) (reconcile.Result, error) { + targetClient, found := s.targets[req.clusterName] + if !found { + return reconcile.Result{}, reconcile.TerminalError(fmt.Errorf("target cluster %s not found", req.clusterName)) + } + + var reference corev1.Secret + if err := s.source.Get(ctx, req.NamespacedName, &reference); err != nil { + if !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to get secret %s from reference cluster: %w", req.String(), err) + } + if err := targetClient.Delete(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: req.Name, + Namespace: targetNamespace, + }}); err != nil { + if !apierrors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to delete secret %s/%s in cluster %s: %w", targetNamespace, req.Name, req.clusterName, err) + } + + return reconcile.Result{}, nil + } + + log.FromContext(ctx).Info("Deleted secret", "cluster", req.clusterName, "namespace", targetNamespace, "name", req.Name) + return reconcile.Result{}, nil + } + + target := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: reference.Name, + Namespace: targetNamespace, + }} + result, err := controllerutil.CreateOrUpdate(ctx, targetClient, target, func() error { + target.Data = reference.Data + return nil + }) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to upsert target secret %s/%s: %w", target.Namespace, target.Name, err) + } + + if result != controllerutil.OperationResultNone { + log.FromContext(ctx).Info("Upserted secret", "cluster", req.clusterName, "namespace", targetNamespace, "name", req.Name, "result", result) + } + + return reconcile.Result{}, nil +} diff --git a/examples/priorityqueue/main.go b/examples/priorityqueue/main.go new file mode 100644 index 0000000000..1dc10c2cbe --- /dev/null +++ b/examples/priorityqueue/main.go @@ -0,0 +1,76 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "os" + "time" + + "go.uber.org/zap/zapcore" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/builder" + kubeconfig "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func init() { +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func run() error { + log.SetLogger(zap.New(func(o *zap.Options) { + o.Level = zapcore.Level(-5) + })) + + // Setup a Manager + mgr, err := manager.New(kubeconfig.GetConfigOrDie(), manager.Options{ + Controller: config.Controller{}, + }) + if err != nil { + return fmt.Errorf("failed to set up controller-manager: %w", err) + } + + if err := builder.ControllerManagedBy(mgr). + For(&corev1.ConfigMap{}). + Complete(reconcile.Func(func(ctx context.Context, r reconcile.Request) (reconcile.Result, error) { + log.FromContext(ctx).Info("Reconciling") + time.Sleep(10 * time.Second) + + return reconcile.Result{}, nil + })); err != nil { + return fmt.Errorf("failed to set up controller: %w", err) + } + + if err := mgr.Start(signals.SetupSignalHandler()); err != nil { + return fmt.Errorf("failed to start manager: %w", err) + } + + return nil +} diff --git a/examples/scratch-env/go.mod b/examples/scratch-env/go.mod index 59da6b3d9e..546c7c39ee 100644 --- a/examples/scratch-env/go.mod +++ b/examples/scratch-env/go.mod @@ -1,10 +1,70 @@ module sigs.k8s.io/controller-runtime/examples/scratch-env -go 1.15 +go 1.24.0 require ( - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6 + go.uber.org/zap v1.27.0 sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 ) +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.34.1 // indirect + k8s.io/apiextensions-apiserver v0.34.1 // indirect + k8s.io/apimachinery v0.34.1 // indirect + k8s.io/client-go v0.34.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) + replace sigs.k8s.io/controller-runtime => ../.. diff --git a/examples/scratch-env/go.sum b/examples/scratch-env/go.sum index ffafe85781..012b88f447 100644 --- a/examples/scratch-env/go.sum +++ b/examples/scratch-env/go.sum @@ -1,840 +1,197 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.12.0 h1:p4oGGk2M2UJc0wWN4lHFvIB71lxsh0T/UiKCCgFADY8= -github.com/onsi/gomega v1.12.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg= -github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 h1:xUIPaMhvROX9dhPvRCenIJtU78+lbEenGbgqB5hfHCQ= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c= -k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= -k8s.io/apiextensions-apiserver v0.21.1 h1:AA+cnsb6w7SZ1vD32Z+zdgfXdXY8X9uGX5bN6EoPEIo= -k8s.io/apiextensions-apiserver v0.21.1/go.mod h1:KESQFCGjqVcVsZ9g0xX5bacMjyX5emuWcS2arzdEouA= -k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs= -k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= -k8s.io/apiserver v0.21.1/go.mod h1:nLLYZvMWn35glJ4/FZRhzLG/3MPxAaZTgV4FJZdr+tY= -k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4= -k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= -k8s.io/code-generator v0.21.1/go.mod h1:hUlps5+9QaTrKx+jiM4rmq7YmH8wPOIko64uZCHDh6Q= -k8s.io/component-base v0.21.1 h1:iLpj2btXbR326s/xNQWmPNGu0gaYSjzn7IN/5i28nQw= -k8s.io/component-base v0.21.1/go.mod h1:NgzFZ2qu4m1juby4TnrmpR8adRk6ka62YdH5DkIIyKA= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210517184530-5a248b5acedc h1:cIS13bDBZaWqngldgGuDypv4z+zjcYgTKv72k6bMAn0= -k8s.io/utils v0.0.0-20210517184530-5a248b5acedc/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= +k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/examples/tokenreview/tokenreview.go b/examples/tokenreview/tokenreview.go index cc64545e16..16e4151077 100644 --- a/examples/tokenreview/tokenreview.go +++ b/examples/tokenreview/tokenreview.go @@ -28,7 +28,7 @@ import ( type authenticator struct { } -// authenticator admits a request by the token. +// Handle admits a request by the token. func (a *authenticator) Handle(ctx context.Context, req authentication.Request) authentication.Response { if req.Spec.Token == "invalid" { return authentication.Unauthenticated("invalid is an invalid token", v1.UserInfo{}) diff --git a/examples/typed/main.go b/examples/typed/main.go new file mode 100644 index 0000000000..7245ce844d --- /dev/null +++ b/examples/typed/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "fmt" + "os" + + networkingv1 "k8s.io/api/networking/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func run() error { + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{}) + if err != nil { + return fmt.Errorf("failed to construct manager: %w", err) + } + + // Use a request type that is always equal to itself so the workqueue + // de-duplicates all events. + // This can for example be useful for an ingress-controller that + // generates a config from all ingresses, rather than individual ones. + type request struct{} + + r := reconcile.TypedFunc[request](func(ctx context.Context, _ request) (reconcile.Result, error) { + ingressList := &networkingv1.IngressList{} + if err := mgr.GetClient().List(ctx, ingressList); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to list ingresses: %w", err) + } + + buildIngressConfig(ingressList) + return reconcile.Result{}, nil + }) + if err := builder.TypedControllerManagedBy[request](mgr). + WatchesRawSource(source.TypedKind( + mgr.GetCache(), + &networkingv1.Ingress{}, + handler.TypedEnqueueRequestsFromMapFunc(func(context.Context, *networkingv1.Ingress) []request { + return []request{{}} + })), + ). + Named("ingress_controller"). + Complete(r); err != nil { + return fmt.Errorf("failed to construct ingress-controller: %w", err) + } + + ctx := signals.SetupSignalHandler() + if err := mgr.Start(ctx); err != nil { + return fmt.Errorf("failed to start manager: %w", err) + } + + return nil +} + +func buildIngressConfig(*networkingv1.IngressList) {} diff --git a/go.mod b/go.mod index a5d7da8406..4d998fe2fc 100644 --- a/go.mod +++ b/go.mod @@ -1,74 +1,103 @@ module sigs.k8s.io/controller-runtime -go 1.20 +go 1.24.0 require ( - github.com/evanphx/json-patch v5.6.0+incompatible - github.com/evanphx/json-patch/v5 v5.6.0 - github.com/fsnotify/fsnotify v1.6.0 - github.com/go-logr/logr v1.2.4 - github.com/go-logr/zapr v1.2.4 - github.com/google/go-cmp v0.5.9 - github.com/onsi/ginkgo/v2 v2.11.0 - github.com/onsi/gomega v1.27.8 - github.com/prometheus/client_golang v1.16.0 - github.com/prometheus/client_model v0.4.0 - go.uber.org/goleak v1.2.1 - go.uber.org/zap v1.24.0 - golang.org/x/sys v0.10.0 - gomodules.xyz/jsonpatch/v2 v2.3.0 - k8s.io/api v0.28.0-alpha.3 - k8s.io/apiextensions-apiserver v0.28.0-alpha.3 - k8s.io/apimachinery v0.28.0-alpha.3 - k8s.io/client-go v0.28.0-alpha.3 - k8s.io/component-base v0.28.0-alpha.3 - k8s.io/klog/v2 v2.100.1 - k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 - sigs.k8s.io/yaml v1.3.0 + github.com/evanphx/json-patch/v5 v5.9.11 + github.com/fsnotify/fsnotify v1.9.0 + github.com/go-logr/logr v1.4.2 + github.com/go-logr/zapr v1.3.0 + github.com/google/btree v1.1.3 + github.com/google/go-cmp v0.7.0 + github.com/google/gofuzz v1.2.0 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + github.com/prometheus/client_golang v1.22.0 + github.com/prometheus/client_model v0.6.1 + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 + golang.org/x/mod v0.21.0 + golang.org/x/sync v0.12.0 + golang.org/x/sys v0.31.0 + gomodules.xyz/jsonpatch/v2 v2.4.0 + gopkg.in/evanphx/json-patch.v4 v4.12.0 // Using v4 to match upstream + k8s.io/api v0.34.1 + k8s.io/apiextensions-apiserver v0.34.1 + k8s.io/apimachinery v0.34.1 + k8s.io/apiserver v0.34.1 + k8s.io/client-go v0.34.1 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 + sigs.k8s.io/yaml v1.6.0 ) require ( + cel.dev/expr v0.24.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/imdario/mergo v0.3.6 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - go.uber.org/atomic v1.10.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.34.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/term v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/grpc v1.72.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + k8s.io/component-base v0.34.1 // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index cee66b77d3..d6278d8a7d 100644 --- a/go.sum +++ b/go.sum @@ -1,225 +1,259 @@ -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.3.0 h1:8NFhfS6gzxNqjLIYnZxg319wZ5Qjnx4m/CcX+Klzazc= -gomodules.xyz/jsonpatch/v2 v2.3.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.28.0-alpha.3 h1:dl9ku8PmbvD7VFdgYzA34SRGsg5hpmDeqZUC670kVjw= -k8s.io/api v0.28.0-alpha.3/go.mod h1:8xUcnnu+0XlJiU0p9mwS7tRpUB1ce+XZrCYsEHeUQRw= -k8s.io/apiextensions-apiserver v0.28.0-alpha.3 h1:gLXUken0/NqJJdVSmTIyuGdEvynVCuUjf3rPhPn2rGo= -k8s.io/apiextensions-apiserver v0.28.0-alpha.3/go.mod h1:L7BBVLOInbK+bFapKaIRzgqruDatxfQO8mJvzBmeh/A= -k8s.io/apimachinery v0.28.0-alpha.3 h1:poqReta738jpeYUyvxXxYbOk6hj5vc1EcKxyo0zhklk= -k8s.io/apimachinery v0.28.0-alpha.3/go.mod h1:tAiIbF8KB8+Ri2DfUWwZGwNOThIwM0fhXLnOymriu+4= -k8s.io/client-go v0.28.0-alpha.3 h1:CPcmGXW3lRogryie7htr93ZIitk6H+zrzPns3TpuFyY= -k8s.io/client-go v0.28.0-alpha.3/go.mod h1:1KTCIXQZjy18QNeXI6zDkq1w2E4UP6gjl1fp5QRF3mo= -k8s.io/component-base v0.28.0-alpha.3 h1:9YtZ5WToliqoAeVcKyN2AHSG4D78E144OQGEJOussjk= -k8s.io/component-base v0.28.0-alpha.3/go.mod h1:JCw6Ny1cRI7y5WBjIgqvdTFEuiyZBHWDORST3BwvQpg= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961 h1:pqRVJGQJz6oeZby8qmPKXYIBjyrcv7EHCe/33UkZMYA= -k8s.io/kube-openapi v0.0.0-20230601164746-7562a1006961/go.mod h1:l8HTwL5fqnlns4jOveW1L75eo7R9KFHxiE0bsPGy428= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= -k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= -sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM= +k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk= +k8s.io/apiextensions-apiserver v0.34.1 h1:NNPBva8FNAPt1iSVwIE0FsdrVriRXMsaWFMqJbII2CI= +k8s.io/apiextensions-apiserver v0.34.1/go.mod h1:hP9Rld3zF5Ay2Of3BeEpLAToP+l4s5UlxiHfqRaRcMc= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apiserver v0.34.1 h1:U3JBGdgANK3dfFcyknWde1G6X1F4bg7PXuvlqt8lITA= +k8s.io/apiserver v0.34.1/go.mod h1:eOOc9nrVqlBI1AFCvVzsob0OxtPZUCPiUJL45JOTBG0= +k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= +k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/component-base v0.34.1 h1:v7xFgG+ONhytZNFpIz5/kecwD+sUhVE6HU7qQUiRM4A= +k8s.io/component-base v0.34.1/go.mod h1:mknCpLlTSKHzAQJJnnHVKqjxR7gBeHRv0rPXA7gdtQ0= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/apidiff.sh b/hack/apidiff.sh index 0167486da1..a15342d16a 100755 --- a/hack/apidiff.sh +++ b/hack/apidiff.sh @@ -23,13 +23,8 @@ source $(dirname ${BASH_SOURCE})/common.sh REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. cd "${REPO_ROOT}" -APIDIFF="hack/tools/bin/go-apidiff" - -header_text "fetching and building go-apidiff" -make "${APIDIFF}" - -git status +export GOTOOLCHAIN="go$(make --silent go-version)" header_text "verifying api diff" -header_text "invoking: '${APIDIFF} ${PULL_BASE_SHA} --print-compatible'" -"${APIDIFF}" "${PULL_BASE_SHA}" --print-compatible +echo "*** Running go-apidiff ***" +APIDIFF_OLD_COMMIT="${PULL_BASE_SHA}" make verify-apidiff diff --git a/hack/check-everything.sh b/hack/check-everything.sh index 09f2a901d4..84db032176 100755 --- a/hack/check-everything.sh +++ b/hack/check-everything.sh @@ -24,11 +24,13 @@ source ${hack_dir}/common.sh tmp_root=/tmp kb_root_dir=$tmp_root/kubebuilder +export GOTOOLCHAIN="go$(make --silent go-version)" + # Run verification scripts. ${hack_dir}/verify.sh # Envtest. -ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.24.2"} +ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.32.0"} header_text "installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest if necessary" tmp_bin=/tmp/cr-tests-bin @@ -45,8 +47,6 @@ ${hack_dir}/test-all.sh header_text "confirming examples compile (via go install)" go install ${MOD_OPT} ./examples/builtins go install ${MOD_OPT} ./examples/crd -go install ${MOD_OPT} ./examples/configfile/builtin -go install ${MOD_OPT} ./examples/configfile/custom echo "passed" exit 0 diff --git a/hack/ensure-golangci-lint.sh b/hack/ensure-golangci-lint.sh deleted file mode 100755 index 9210f959b0..0000000000 --- a/hack/ensure-golangci-lint.sh +++ /dev/null @@ -1,432 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE: This script is copied from from https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh. - -set -e - -usage() { - this=$1 - cat </dev/null -} -echoerr() { - echo "$@" 1>&2 -} -log_prefix() { - echo "$0" -} -_logp=6 -log_set_priority() { - _logp="$1" -} -log_priority() { - if test -z "$1"; then - echo "$_logp" - return - fi - [ "$1" -le "$_logp" ] -} -log_tag() { - case $1 in - 0) echo "emerg" ;; - 1) echo "alert" ;; - 2) echo "crit" ;; - 3) echo "err" ;; - 4) echo "warning" ;; - 5) echo "notice" ;; - 6) echo "info" ;; - 7) echo "debug" ;; - *) echo "$1" ;; - esac -} -log_debug() { - log_priority 7 || return 0 - echoerr "$(log_prefix)" "$(log_tag 7)" "$@" -} -log_info() { - log_priority 6 || return 0 - echoerr "$(log_prefix)" "$(log_tag 6)" "$@" -} -log_err() { - log_priority 3 || return 0 - echoerr "$(log_prefix)" "$(log_tag 3)" "$@" -} -log_crit() { - log_priority 2 || return 0 - echoerr "$(log_prefix)" "$(log_tag 2)" "$@" -} -uname_os() { - os=$(uname -s | tr '[:upper:]' '[:lower:]') - case "$os" in - msys*) os="windows" ;; - mingw*) os="windows" ;; - cygwin*) os="windows" ;; - win*) os="windows" ;; - esac - echo "$os" -} -uname_arch() { - arch=$(uname -m) - case $arch in - x86_64) arch="amd64" ;; - x86) arch="386" ;; - i686) arch="386" ;; - i386) arch="386" ;; - aarch64) arch="arm64" ;; - armv5*) arch="armv5" ;; - armv6*) arch="armv6" ;; - armv7*) arch="armv7" ;; - esac - echo ${arch} -} -uname_os_check() { - os=$(uname_os) - case "$os" in - darwin) return 0 ;; - dragonfly) return 0 ;; - freebsd) return 0 ;; - linux) return 0 ;; - android) return 0 ;; - nacl) return 0 ;; - netbsd) return 0 ;; - openbsd) return 0 ;; - plan9) return 0 ;; - solaris) return 0 ;; - windows) return 0 ;; - esac - log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value." - return 1 -} -uname_arch_check() { - arch=$(uname_arch) - case "$arch" in - 386) return 0 ;; - amd64) return 0 ;; - arm64) return 0 ;; - armv5) return 0 ;; - armv6) return 0 ;; - armv7) return 0 ;; - ppc64) return 0 ;; - ppc64le) return 0 ;; - mips) return 0 ;; - mipsle) return 0 ;; - mips64) return 0 ;; - mips64le) return 0 ;; - s390x) return 0 ;; - riscv64) return 0 ;; - amd64p32) return 0 ;; - esac - log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value." - return 1 -} -untar() { - tarball=$1 - case "${tarball}" in - *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;; - *.tar) tar --no-same-owner -xf "${tarball}" ;; - *.zip) unzip "${tarball}" ;; - *) - log_err "untar unknown archive format for ${tarball}" - return 1 - ;; - esac -} -http_download_curl() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url") - else - code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url") - fi - if [ "$code" != "200" ]; then - log_debug "http_download_curl received HTTP status $code" - return 1 - fi - return 0 -} -http_download_wget() { - local_file=$1 - source_url=$2 - header=$3 - if [ -z "$header" ]; then - wget -q -O "$local_file" "$source_url" - else - wget -q --header "$header" -O "$local_file" "$source_url" - fi -} -http_download() { - log_debug "http_download $2" - if is_command curl; then - http_download_curl "$@" - return - elif is_command wget; then - http_download_wget "$@" - return - fi - log_crit "http_download unable to find wget or curl" - return 1 -} -http_copy() { - tmp=$(mktemp) - http_download "${tmp}" "$1" "$2" || return 1 - body=$(cat "$tmp") - rm -f "${tmp}" - echo "$body" -} -github_release() { - owner_repo=$1 - version=$2 - if [ -z "$version" ]; then - giturl="https://api.github.com/repos/${owner_repo}/releases/latest" - else - giturl="https://api.github.com/repos/${owner_repo}/releases/tags/${version}" - fi - json=$(http_copy "$giturl" "Accept:application/json") - test -z "$json" && return 1 - version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name": "//' | sed 's/".*//') - test -z "$version" && return 1 - echo "$version" -} -hash_sha256() { - TARGET=${1:-/dev/stdin} - if is_command gsha256sum; then - hash=$(gsha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command sha256sum; then - hash=$(sha256sum "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command shasum; then - hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1 - echo "$hash" | cut -d ' ' -f 1 - elif is_command openssl; then - hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1 - echo "$hash" | cut -d ' ' -f a - else - log_crit "hash_sha256 unable to find command to compute sha-256 hash" - return 1 - fi -} -hash_sha256_verify() { - TARGET=$1 - checksums=$2 - if [ -z "$checksums" ]; then - log_err "hash_sha256_verify checksum file not specified in arg2" - return 1 - fi - BASENAME=${TARGET##*/} - want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1) - if [ -z "$want" ]; then - log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'" - return 1 - fi - got=$(hash_sha256 "$TARGET") - if [ "$want" != "$got" ]; then - log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got" - return 1 - fi -} -cat /dev/null < 0 { - return nil, errors.New("number of replica should be less than or equal to 0 to delete") - } - return nil, nil -} - // TestDefaultValidator. var _ runtime.Object = &TestDefaultValidator{} +const testDefaultValidatorKind = "TestDefaultValidator" + type TestDefaultValidator struct { metav1.TypeMeta metav1.ObjectMeta @@ -816,37 +1035,7 @@ type TestDefaultValidatorList struct{} func (*TestDefaultValidatorList) GetObjectKind() schema.ObjectKind { return nil } func (*TestDefaultValidatorList) DeepCopyObject() runtime.Object { return nil } -func (dv *TestDefaultValidator) Default() { - if dv.Replica < 2 { - dv.Replica = 2 - } -} - -var _ admission.Validator = &TestDefaultValidator{} - -func (dv *TestDefaultValidator) ValidateCreate() (admission.Warnings, error) { - if dv.Replica < 0 { - return nil, errors.New("number of replica should be greater than or equal to 0") - } - return nil, nil -} - -func (dv *TestDefaultValidator) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { - if dv.Replica < 0 { - return nil, errors.New("number of replica should be greater than or equal to 0") - } - return nil, nil -} - -func (dv *TestDefaultValidator) ValidateDelete() (admission.Warnings, error) { - if dv.Replica > 0 { - return nil, errors.New("number of replica should be less than or equal to 0 to delete") - } - return nil, nil -} - // TestCustomDefaulter. - type TestCustomDefaulter struct{} func (*TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) error { @@ -860,9 +1049,14 @@ func (*TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) err } d := obj.(*TestDefaulter) //nolint:ifshort + if d.Panic { + panic("fake panic test") + } + if d.Replica < 2 { d.Replica = 2 } + return nil } @@ -883,9 +1077,13 @@ func (*TestCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Obje } v := obj.(*TestValidator) //nolint:ifshort + if v.Panic { + panic("fake panic test") + } if v.Replica < 0 { return nil, errors.New("number of replica should be greater than or equal to 0") } + return nil, nil } @@ -907,6 +1105,12 @@ func (*TestCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj r if v.Replica < old.Replica { return nil, fmt.Errorf("new replica %v should not be fewer than old replica %v", v.Replica, old.Replica) } + + userAgent, ok := ctx.Value(userAgentCtxKey).(string) + if ok && userAgent != userAgentValue { + return nil, fmt.Errorf("expected %s value is %q in TestCustomValidator got %q", userAgentCtxKey, userAgentValue, userAgent) + } + return nil, nil } @@ -928,3 +1132,85 @@ func (*TestCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Obje } var _ admission.CustomValidator = &TestCustomValidator{} + +// TestCustomDefaultValidator for default +type TestCustomDefaultValidator struct{} + +func (*TestCustomDefaultValidator) Default(ctx context.Context, obj runtime.Object) error { + logf.FromContext(ctx).Info("Defaulting object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } + + d := obj.(*TestDefaultValidator) //nolint:ifshort + + if d.Replica < 2 { + d.Replica = 2 + } + return nil +} + +var _ admission.CustomDefaulter = &TestCustomDefaulter{} + +// TestCustomDefaultValidator for validation + +func (*TestCustomDefaultValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return nil, fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } + + v := obj.(*TestDefaultValidator) //nolint:ifshort + if v.Replica < 0 { + return nil, errors.New("number of replica should be greater than or equal to 0") + } + return nil, nil +} + +func (*TestCustomDefaultValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return nil, fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } + + v := newObj.(*TestDefaultValidator) + old := oldObj.(*TestDefaultValidator) + if v.Replica < 0 { + return nil, errors.New("number of replica should be greater than or equal to 0") + } + if v.Replica < old.Replica { + return nil, fmt.Errorf("new replica %v should not be fewer than old replica %v", v.Replica, old.Replica) + } + return nil, nil +} + +func (*TestCustomDefaultValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaultValidatorKind { + return nil, fmt.Errorf("expected Kind TestDefaultValidator got %q", req.Kind.Kind) + } + + v := obj.(*TestDefaultValidator) //nolint:ifshort + if v.Replica > 0 { + return nil, errors.New("number of replica should be less than or equal to 0 to delete") + } + return nil, nil +} + +var _ admission.CustomValidator = &TestCustomValidator{} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index f01de43810..a7e491855a 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -19,9 +19,13 @@ package cache import ( "context" "fmt" + "maps" "net/http" + "slices" + "sort" "time" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -31,26 +35,39 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache/internal" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) var ( - log = logf.RuntimeLog.WithName("object-cache") defaultSyncPeriod = 10 * time.Hour ) +// InformerGetOptions defines the behavior of how informers are retrieved. +type InformerGetOptions internal.GetOptions + +// InformerGetOption defines an option that alters the behavior of how informers are retrieved. +type InformerGetOption func(*InformerGetOptions) + +// BlockUntilSynced determines whether a get request for an informer should block +// until the informer's cache has synced. +func BlockUntilSynced(shouldBlock bool) InformerGetOption { + return func(opts *InformerGetOptions) { + opts.BlockUntilSynced = &shouldBlock + } +} + // Cache knows how to load Kubernetes objects, fetch informers to request // to receive events for Kubernetes objects (at a low-level), // and add indices to fields on the objects stored in the cache. type Cache interface { - // Cache acts as a client to objects stored in the cache. + // Reader acts as a client to objects stored in the cache. client.Reader - // Cache loads informers and adds field indices. + // Informers loads informers and adds field indices. Informers } @@ -60,49 +77,66 @@ type Cache interface { type Informers interface { // GetInformer fetches or constructs an informer for the given object that corresponds to a single // API kind and resource. - GetInformer(ctx context.Context, obj client.Object) (Informer, error) + GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) // GetInformerForKind is similar to GetInformer, except that it takes a group-version-kind, instead // of the underlying object. - GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) + GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) + + // RemoveInformer removes an informer entry and stops it if it was running. + RemoveInformer(ctx context.Context, obj client.Object) error // Start runs all the informers known to this cache until the context is closed. // It blocks. Start(ctx context.Context) error - // WaitForCacheSync waits for all the caches to sync. Returns false if it could not sync a cache. + // WaitForCacheSync waits for all the caches to sync. Returns false if it could not sync a cache. WaitForCacheSync(ctx context.Context) bool - // Informers knows how to add indices to the caches (informers) that it manages. + // FieldIndexer adds indices to the managed informers. client.FieldIndexer } -// Informer - informer allows you interact with the underlying informer. +// Informer allows you to interact with the underlying informer. type Informer interface { // AddEventHandler adds an event handler to the shared informer using the shared informer's resync - // period. Events to a single handler are delivered sequentially, but there is no coordination + // period. Events to a single handler are delivered sequentially, but there is no coordination // between different handlers. // It returns a registration handle for the handler that can be used to remove - // the handler again. + // the handler again and an error if the handler cannot be added. AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) + // AddEventHandlerWithResyncPeriod adds an event handler to the shared informer using the - // specified resync period. Events to a single handler are delivered sequentially, but there is + // specified resync period. Events to a single handler are delivered sequentially, but there is // no coordination between different handlers. // It returns a registration handle for the handler that can be used to remove // the handler again and an error if the handler cannot be added. AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) - // RemoveEventHandler removes a formerly added event handler given by + + // AddEventHandlerWithOptions is a variant of AddEventHandlerWithResyncPeriod where + // all optional parameters are passed in as a struct. + AddEventHandlerWithOptions(handler toolscache.ResourceEventHandler, options toolscache.HandlerOptions) (toolscache.ResourceEventHandlerRegistration, error) + + // RemoveEventHandler removes a previously added event handler given by // its registration handle. - // This function is guaranteed to be idempotent, and thread-safe. + // This function is guaranteed to be idempotent and thread-safe. RemoveEventHandler(handle toolscache.ResourceEventHandlerRegistration) error - // AddIndexers adds more indexers to this store. If you call this after you already have data - // in the store, the results are undefined. + + // AddIndexers adds indexers to this store. It is valid to add indexers + // after an informer was started. AddIndexers(indexers toolscache.Indexers) error + // HasSynced return true if the informers underlying store has synced. HasSynced() bool + // IsStopped returns true if the informer has been stopped. + IsStopped() bool } -// Options are the optional arguments for creating a new InformersMap object. +// AllNamespaces should be used as the map key to deliminate namespace settings +// that apply to all namespaces that themselves do not have explicit settings. +const AllNamespaces = metav1.NamespaceAll + +// Options are the optional arguments for creating a new Cache object. type Options struct { // HTTPClient is the http client to use for the REST client HTTPClient *http.Client @@ -138,47 +172,123 @@ type Options struct { // is "done" with an object, and would otherwise not requeue it, i.e., we // recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`, // instead of `reconcile.Result{}`. + // + // SyncPeriod will locally trigger an artificial Update event with the same + // object in both ObjectOld and ObjectNew for everything that is in the + // cache. + // + // Predicates or Handlers that expect ObjectOld and ObjectNew to be different + // (such as GenerationChangedPredicate) will filter out this event, preventing + // it from triggering a reconciliation. + // SyncPeriod does not sync between the local cache and the server. SyncPeriod *time.Duration - // Namespaces restricts the cache's ListWatch to the desired namespaces - // Default watches all namespaces - Namespaces []string + // ReaderFailOnMissingInformer configures the cache to return a ErrResourceNotCached error when a user + // requests, using Get() and List(), a resource the cache does not already have an informer for. + // + // This error is distinct from an errors.NotFound. + // + // Defaults to false, which means that the cache will start a new informer + // for every new requested resource. + ReaderFailOnMissingInformer bool - // DefaultLabelSelector will be used as a label selectors for all object types - // unless they have a more specific selector set in ByObject. + // DefaultNamespaces maps namespace names to cache configs. If set, only + // the namespaces in here will be watched and it will by used to default + // ByObject.Namespaces for all objects if that is nil. + // + // It is possible to have specific Config for just some namespaces + // but cache all namespaces by using the AllNamespaces const as the map key. + // This will then include all namespaces that do not have a more specific + // setting. + // + // The options in the Config that are nil will be defaulted from + // the respective Default* settings. + DefaultNamespaces map[string]Config + + // DefaultLabelSelector will be used as a label selector for all objects + // unless there is already one set in ByObject or DefaultNamespaces. DefaultLabelSelector labels.Selector - // DefaultFieldSelector will be used as a field selectors for all object types - // unless they have a more specific selector set in ByObject. + // DefaultFieldSelector will be used as a field selector for all object types + // unless there is already one set in ByObject or DefaultNamespaces. DefaultFieldSelector fields.Selector // DefaultTransform will be used as transform for all object types - // unless they have a more specific transform set in ByObject. + // unless there is already one set in ByObject or DefaultNamespaces. + // + // A typical usecase for this is to use TransformStripManagedFields + // to reduce the caches memory usage. DefaultTransform toolscache.TransformFunc - // ByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object. - ByObject map[client.Object]ByObject + // DefaultWatchErrorHandler will be used to set the WatchErrorHandler which is called + // whenever ListAndWatch drops the connection with an error. + // + // After calling this handler, the informer will backoff and retry. + DefaultWatchErrorHandler toolscache.WatchErrorHandlerWithContext - // UnsafeDisableDeepCopy indicates not to deep copy objects during get or - // list objects for EVERY object. + // DefaultUnsafeDisableDeepCopy is the default for UnsafeDisableDeepCopy + // for everything that doesn't specify this. + // // Be very careful with this, when enabled you must DeepCopy any object before mutating it, // otherwise you will mutate the object in the cache. // - // This is a global setting for all objects, and can be overridden by the ByObject setting. - UnsafeDisableDeepCopy *bool + // This will be used for all object types, unless it is set in ByObject or + // DefaultNamespaces. + DefaultUnsafeDisableDeepCopy *bool + + // DefaultEnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // This will be used for all object types, unless it is set in ByObject or + // DefaultNamespaces. + // + // Defaults to true. + DefaultEnableWatchBookmarks *bool + + // ByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object. + // If unset, this will fall through to the Default* settings. + ByObject map[client.Object]ByObject + + // NewInformer allows overriding of NewSharedIndexInformer, for example for testing + // or if someone wants to write their own Informer. + NewInformer func(toolscache.ListerWatcher, runtime.Object, time.Duration, toolscache.Indexers) toolscache.SharedIndexInformer } // ByObject offers more fine-grained control over the cache's ListWatch by object. type ByObject struct { + // Namespaces maps a namespace name to cache configs. If set, only the + // namespaces in this map will be cached. + // + // Settings in the map value that are unset will be defaulted. + // Use an empty value for the specific setting to prevent that. + // + // It is possible to have specific Config for just some namespaces + // but cache all namespaces by using the AllNamespaces const as the map key. + // This will then include all namespaces that do not have a more specific + // setting. + // + // A nil map allows to default this to the cache's DefaultNamespaces setting. + // An empty map prevents this and means that all namespaces will be cached. + // + // The defaulting follows the following precedence order: + // 1. ByObject + // 2. DefaultNamespaces[namespace] + // 3. Default* + // + // This must be unset for cluster-scoped objects. + Namespaces map[string]Config + // Label represents a label selector for the object. Label labels.Selector // Field represents a field selector for the object. Field fields.Selector - // Transform is a map from objects to transformer functions which - // get applied when objects of the transformation are about to be committed - // to cache. + // Transform is a transformer function for the object which gets applied + // when objects of the transformation are about to be committed to the cache. // // This function is called both for new objects to enter the cache, // and for updated objects. @@ -189,62 +299,172 @@ type ByObject struct { // Be very careful with this, when enabled you must DeepCopy any object before mutating it, // otherwise you will mutate the object in the cache. UnsafeDisableDeepCopy *bool + + // EnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // Defaults to true. + EnableWatchBookmarks *bool +} + +// Config describes all potential options for a given watch. +type Config struct { + // LabelSelector specifies a label selector. A nil value allows to + // default this. + // + // Set to labels.Everything() if you don't want this defaulted. + LabelSelector labels.Selector + + // FieldSelector specifics a field selector. A nil value allows to + // default this. + // + // Set to fields.Everything() if you don't want this defaulted. + FieldSelector fields.Selector + + // Transform specifies a transform func. A nil value allows to default + // this. + // + // Set to an empty func to prevent this: + // func(in interface{}) (interface{}, error) { return in, nil } + Transform toolscache.TransformFunc + + // UnsafeDisableDeepCopy specifies if List and Get requests against the + // cache should not DeepCopy. A nil value allows to default this. + UnsafeDisableDeepCopy *bool + + // EnableWatchBookmarks requests watch events with type "BOOKMARK". + // Servers that do not implement bookmarks may ignore this flag and + // bookmarks are sent at the server's discretion. Clients should not + // assume bookmarks are returned at any specific interval, nor may they + // assume the server will send any BOOKMARK event during a session. + // + // Defaults to true. + EnableWatchBookmarks *bool } // NewCacheFunc - Function for creating a new cache from the options and a rest config. type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error) // New initializes and returns a new Cache. -func New(config *rest.Config, opts Options) (Cache, error) { - if len(opts.Namespaces) == 0 { - opts.Namespaces = []string{metav1.NamespaceAll} +func New(cfg *rest.Config, opts Options) (Cache, error) { + opts, err := defaultOpts(cfg, opts) + if err != nil { + return nil, err } - if len(opts.Namespaces) > 1 { - return newMultiNamespaceCache(config, opts) + + newCacheFunc := newCache(cfg, opts) + + var defaultCache Cache + if len(opts.DefaultNamespaces) > 0 { + defaultConfig := optionDefaultsToConfig(&opts) + defaultCache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, opts.DefaultNamespaces, &defaultConfig) + } else { + defaultCache = newCacheFunc(optionDefaultsToConfig(&opts), corev1.NamespaceAll) } - opts, err := defaultOpts(config, opts) - if err != nil { - return nil, err + if len(opts.ByObject) == 0 { + return defaultCache, nil } - byGVK, err := convertToInformerOptsByGVK(opts.ByObject, opts.Scheme) - if err != nil { - return nil, err + delegating := &delegatingByGVKCache{ + scheme: opts.Scheme, + caches: make(map[schema.GroupVersionKind]Cache, len(opts.ByObject)), + defaultCache: defaultCache, + } + + for obj, config := range opts.ByObject { + gvk, err := apiutil.GVKForObject(obj, opts.Scheme) + if err != nil { + return nil, fmt.Errorf("failed to get GVK for type %T: %w", obj, err) + } + var cache Cache + if len(config.Namespaces) > 0 { + cache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, config.Namespaces, nil) + } else { + cache = newCacheFunc(byObjectToConfig(config), corev1.NamespaceAll) + } + delegating.caches[gvk] = cache + } + + return delegating, nil +} + +// TransformStripManagedFields strips the managed fields of an object before it is committed to the cache. +// If you are not explicitly accessing managedFields from your code, setting this as `DefaultTransform` +// on the cache can lead to a significant reduction in memory usage. +func TransformStripManagedFields() toolscache.TransformFunc { + return func(in any) (any, error) { + // Nilcheck managed fields to avoid hitting https://github.com/kubernetes/kubernetes/issues/124337 + if obj, err := meta.Accessor(in); err == nil && obj.GetManagedFields() != nil { + obj.SetManagedFields(nil) + } + + return in, nil } - // Set the default selector and transform. - byGVK[schema.GroupVersionKind{}] = internal.InformersOptsByGVK{ - Selector: internal.Selector{ - Label: opts.DefaultLabelSelector, - Field: opts.DefaultFieldSelector, - }, +} + +func optionDefaultsToConfig(opts *Options) Config { + return Config{ + LabelSelector: opts.DefaultLabelSelector, + FieldSelector: opts.DefaultFieldSelector, Transform: opts.DefaultTransform, - UnsafeDisableDeepCopy: opts.UnsafeDisableDeepCopy, + UnsafeDisableDeepCopy: opts.DefaultUnsafeDisableDeepCopy, + EnableWatchBookmarks: opts.DefaultEnableWatchBookmarks, + } +} + +func byObjectToConfig(byObject ByObject) Config { + return Config{ + LabelSelector: byObject.Label, + FieldSelector: byObject.Field, + Transform: byObject.Transform, + UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy, + EnableWatchBookmarks: byObject.EnableWatchBookmarks, } +} - return &informerCache{ - scheme: opts.Scheme, - Informers: internal.NewInformers(config, &internal.InformersOpts{ - HTTPClient: opts.HTTPClient, - Scheme: opts.Scheme, - Mapper: opts.Mapper, - ResyncPeriod: *opts.SyncPeriod, - Namespace: opts.Namespaces[0], - ByGVK: byGVK, - }), - }, nil +type newCacheFunc func(config Config, namespace string) Cache + +func newCache(restConfig *rest.Config, opts Options) newCacheFunc { + return func(config Config, namespace string) Cache { + return &informerCache{ + scheme: opts.Scheme, + Informers: internal.NewInformers(restConfig, &internal.InformersOpts{ + HTTPClient: opts.HTTPClient, + Scheme: opts.Scheme, + Mapper: opts.Mapper, + ResyncPeriod: *opts.SyncPeriod, + Namespace: namespace, + Selector: internal.Selector{ + Label: config.LabelSelector, + Field: config.FieldSelector, + }, + Transform: config.Transform, + WatchErrorHandler: opts.DefaultWatchErrorHandler, + UnsafeDisableDeepCopy: ptr.Deref(config.UnsafeDisableDeepCopy, false), + EnableWatchBookmarks: ptr.Deref(config.EnableWatchBookmarks, true), + NewInformer: opts.NewInformer, + }), + readerFailOnMissingInformer: opts.ReaderFailOnMissingInformer, + } + } } func defaultOpts(config *rest.Config, opts Options) (Options, error) { - logger := log.WithName("setup") + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } // Use the rest HTTP client for the provided config if unset if opts.HTTPClient == nil { var err error opts.HTTPClient, err = rest.HTTPClientFor(config) if err != nil { - logger.Error(err, "Failed to get HTTP client") - return opts, fmt.Errorf("could not create HTTP client from config: %w", err) + return Options{}, fmt.Errorf("could not create HTTP client from config: %w", err) } } @@ -256,13 +476,85 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { // Construct a new Mapper if unset if opts.Mapper == nil { var err error - opts.Mapper, err = apiutil.NewDiscoveryRESTMapper(config, opts.HTTPClient) + opts.Mapper, err = apiutil.NewDynamicRESTMapper(config, opts.HTTPClient) if err != nil { - logger.Error(err, "Failed to get API Group-Resources") - return opts, fmt.Errorf("could not create RESTMapper from config: %w", err) + return Options{}, fmt.Errorf("could not create RESTMapper from config: %w", err) } } + opts.ByObject = maps.Clone(opts.ByObject) + opts.DefaultNamespaces = maps.Clone(opts.DefaultNamespaces) + for obj, byObject := range opts.ByObject { + isNamespaced, err := apiutil.IsObjectNamespaced(obj, opts.Scheme, opts.Mapper) + if err != nil { + return opts, fmt.Errorf("failed to determine if %T is namespaced: %w", obj, err) + } + if !isNamespaced && byObject.Namespaces != nil { + return opts, fmt.Errorf("type %T is not namespaced, but its ByObject.Namespaces setting is not nil", obj) + } + + if isNamespaced && byObject.Namespaces == nil { + byObject.Namespaces = maps.Clone(opts.DefaultNamespaces) + } else { + byObject.Namespaces = maps.Clone(byObject.Namespaces) + } + + // Default the namespace-level configs first, because they need to use the undefaulted type-level config + // to be able to potentially fall through to settings from DefaultNamespaces. + for namespace, config := range byObject.Namespaces { + // 1. Default from the undefaulted type-level config + config = defaultConfig(config, byObjectToConfig(byObject)) + // 2. Default from the namespace-level config. This was defaulted from the global default config earlier, but + // might not have an entry for the current namespace. + if defaultNamespaceSettings, hasDefaultNamespace := opts.DefaultNamespaces[namespace]; hasDefaultNamespace { + config = defaultConfig(config, defaultNamespaceSettings) + } + + // 3. Default from the global defaults + config = defaultConfig(config, optionDefaultsToConfig(&opts)) + + if namespace == metav1.NamespaceAll { + config.FieldSelector = fields.AndSelectors( + appendIfNotNil( + namespaceAllSelector(slices.Collect(maps.Keys(byObject.Namespaces))), + config.FieldSelector, + )..., + ) + } + + byObject.Namespaces[namespace] = config + } + + // Only default ByObject iself if it isn't namespaced or has no namespaces configured, as only + // then any of this will be honored. + if !isNamespaced || len(byObject.Namespaces) == 0 { + defaultedConfig := defaultConfig(byObjectToConfig(byObject), optionDefaultsToConfig(&opts)) + byObject.Label = defaultedConfig.LabelSelector + byObject.Field = defaultedConfig.FieldSelector + byObject.Transform = defaultedConfig.Transform + byObject.UnsafeDisableDeepCopy = defaultedConfig.UnsafeDisableDeepCopy + byObject.EnableWatchBookmarks = defaultedConfig.EnableWatchBookmarks + } + + opts.ByObject[obj] = byObject + } + + // Default namespaces after byObject has been defaulted, otherwise a namespace without selectors + // will get the `Default` selectors, then get copied to byObject and then not get defaulted from + // byObject, as it already has selectors. + for namespace, cfg := range opts.DefaultNamespaces { + cfg = defaultConfig(cfg, optionDefaultsToConfig(&opts)) + if namespace == metav1.NamespaceAll { + cfg.FieldSelector = fields.AndSelectors( + appendIfNotNil( + namespaceAllSelector(slices.Collect(maps.Keys(opts.DefaultNamespaces))), + cfg.FieldSelector, + )..., + ) + } + opts.DefaultNamespaces[namespace] = cfg + } + // Default the resync period to 10 hours if unset if opts.SyncPeriod == nil { opts.SyncPeriod = &defaultSyncPeriod @@ -270,24 +562,40 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { return opts, nil } -func convertToInformerOptsByGVK(in map[client.Object]ByObject, scheme *runtime.Scheme) (map[schema.GroupVersionKind]internal.InformersOptsByGVK, error) { - out := map[schema.GroupVersionKind]internal.InformersOptsByGVK{} - for object, byObject := range in { - gvk, err := apiutil.GVKForObject(object, scheme) - if err != nil { - return nil, err - } - if _, ok := out[gvk]; ok { - return nil, fmt.Errorf("duplicate cache options for GVK %v, cache.Options.ByObject has multiple types with the same GroupVersionKind", gvk) - } - out[gvk] = internal.InformersOptsByGVK{ - Selector: internal.Selector{ - Field: byObject.Field, - Label: byObject.Label, - }, - Transform: byObject.Transform, - UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy, +func defaultConfig(toDefault, defaultFrom Config) Config { + if toDefault.LabelSelector == nil { + toDefault.LabelSelector = defaultFrom.LabelSelector + } + if toDefault.FieldSelector == nil { + toDefault.FieldSelector = defaultFrom.FieldSelector + } + if toDefault.Transform == nil { + toDefault.Transform = defaultFrom.Transform + } + if toDefault.UnsafeDisableDeepCopy == nil { + toDefault.UnsafeDisableDeepCopy = defaultFrom.UnsafeDisableDeepCopy + } + if toDefault.EnableWatchBookmarks == nil { + toDefault.EnableWatchBookmarks = defaultFrom.EnableWatchBookmarks + } + return toDefault +} + +func namespaceAllSelector(namespaces []string) []fields.Selector { + selectors := make([]fields.Selector, 0, len(namespaces)-1) + sort.Strings(namespaces) + for _, namespace := range namespaces { + if namespace != metav1.NamespaceAll { + selectors = append(selectors, fields.OneTermNotEqualSelector("metadata.namespace", namespace)) } } - return out, nil + + return selectors +} + +func appendIfNotNil[T comparable](a []T, b T) []T { + if b != *new(T) { + return append(a, b) + } + return a } diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 100825854a..7748e2e317 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -18,11 +18,13 @@ package cache_test import ( "context" + "errors" "fmt" "reflect" "sort" "strconv" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -38,20 +40,22 @@ import ( kscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" kcache "k8s.io/client-go/tools/cache" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" ) const testNodeOne = "test-node-1" +const testNodeTwo = "test-node-2" const testNamespaceOne = "test-namespace-1" const testNamespaceTwo = "test-namespace-2" const testNamespaceThree = "test-namespace-3" // TODO(community): Pull these helper functions into testenv. // Restart policy is included to allow indexing on that field. -func createPodWithLabels(name, namespace string, restartPolicy corev1.RestartPolicy, labels map[string]string) client.Object { +func createPodWithLabels(ctx context.Context, name, namespace string, restartPolicy corev1.RestartPolicy, labels map[string]string) client.Object { three := int64(3) if labels == nil { labels = map[string]string{} @@ -71,12 +75,12 @@ func createPodWithLabels(name, namespace string, restartPolicy corev1.RestartPol } cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = cl.Create(context.Background(), pod) + err = cl.Create(ctx, pod) Expect(err).NotTo(HaveOccurred()) return pod } -func createSvc(name, namespace string, cl client.Client) client.Object { +func createSvc(ctx context.Context, name, namespace string, cl client.Client) client.Object { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -86,51 +90,72 @@ func createSvc(name, namespace string, cl client.Client) client.Object { Ports: []corev1.ServicePort{{Port: 1}}, }, } - err := cl.Create(context.Background(), svc) + err := cl.Create(ctx, svc) Expect(err).NotTo(HaveOccurred()) return svc } -func createSA(name, namespace string, cl client.Client) client.Object { +func createSA(ctx context.Context, name, namespace string, cl client.Client) client.Object { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, } - err := cl.Create(context.Background(), sa) + err := cl.Create(ctx, sa) Expect(err).NotTo(HaveOccurred()) return sa } -func createPod(name, namespace string, restartPolicy corev1.RestartPolicy) client.Object { - return createPodWithLabels(name, namespace, restartPolicy, nil) +func createPod(ctx context.Context, name, namespace string, restartPolicy corev1.RestartPolicy) client.Object { + return createPodWithLabels(ctx, name, namespace, restartPolicy, nil) } -func deletePod(pod client.Object) { +func deletePod(ctx context.Context, pod client.Object) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = cl.Delete(context.Background(), pod) + err = cl.Delete(ctx, pod) Expect(err).NotTo(HaveOccurred()) } var _ = Describe("Informer Cache", func() { CacheTest(cache.New, cache.Options{}) + NonBlockingGetTest(cache.New, cache.Options{}) }) + +var _ = Describe("Informer Cache with ReaderFailOnMissingInformer", func() { + CacheTestReaderFailOnMissingInformer(cache.New, cache.Options{ReaderFailOnMissingInformer: true}) +}) + var _ = Describe("Multi-Namespace Informer Cache", func() { - CacheTest(cache.MultiNamespacedCacheBuilder([]string{testNamespaceOne, testNamespaceTwo, "default"}), cache.Options{}) + CacheTest(cache.New, cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + cache.AllNamespaces: {FieldSelector: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne)}, + testNamespaceTwo: {}, + "default": {}, + }, + }) + NonBlockingGetTest(cache.New, cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + cache.AllNamespaces: {FieldSelector: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne)}, + testNamespaceTwo: {}, + "default": {}, + }, + }) }) var _ = Describe("Informer Cache without global DeepCopy", func() { CacheTest(cache.New, cache.Options{ - UnsafeDisableDeepCopy: pointer.Bool(true), + DefaultUnsafeDisableDeepCopy: ptr.To(true), + }) + NonBlockingGetTest(cache.New, cache.Options{ + DefaultUnsafeDisableDeepCopy: ptr.To(true), }) }) var _ = Describe("Cache with transformers", func() { var ( informerCache cache.Cache - informerCacheCtx context.Context informerCacheCancel context.CancelFunc knownPod1 client.Object knownPod2 client.Object @@ -151,28 +176,31 @@ var _ = Describe("Cache with transformers", func() { return "" } - BeforeEach(func() { - informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo Expect(cfg).NotTo(BeNil()) By("creating three pods") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = ensureNode(testNodeOne, cl) + err = ensureNode(ctx, testNodeOne, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceOne, cl) + err = ensureNamespace(ctx, testNamespaceOne, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceTwo, cl) + err = ensureNamespace(ctx, testNamespaceTwo, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceThree, cl) + err = ensureNamespace(ctx, testNamespaceThree, cl) Expect(err).NotTo(HaveOccurred()) // Includes restart policy since these objects are indexed on this field. - knownPod1 = createPod("test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) - knownPod2 = createPod("test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) - knownPod3 = createPodWithLabels("test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) - knownPod4 = createPodWithLabels("test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) - knownPod5 = createPod("test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) - knownPod6 = createPod("test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod1 = createPod(ctx, "test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) + knownPod2 = createPod(ctx, "test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod3 = createPodWithLabels(ctx, "test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) + knownPod4 = createPodWithLabels(ctx, "test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) + knownPod5 = createPod(ctx, "test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) + knownPod6 = createPod(ctx, "test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) podGVK := schema.GroupVersionKind{ Kind: "Pod", @@ -239,26 +267,26 @@ var _ = Describe("Cache with transformers", func() { defer GinkgoRecover() Expect(informerCache.Start(ctx)).To(Succeed()) }(informerCacheCtx) - Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { By("cleaning up created pods") - deletePod(knownPod1) - deletePod(knownPod2) - deletePod(knownPod3) - deletePod(knownPod4) - deletePod(knownPod5) - deletePod(knownPod6) + deletePod(ctx, knownPod1) + deletePod(ctx, knownPod2) + deletePod(ctx, knownPod3) + deletePod(ctx, knownPod4) + deletePod(ctx, knownPod5) + deletePod(ctx, knownPod6) informerCacheCancel() }) Context("with structured objects", func() { - It("should apply transformers to explicitly specified GVKS", func() { + It("should apply transformers to explicitly specified GVKS", func(ctx SpecContext) { By("listing pods") out := corev1.PodList{} - Expect(informerCache.List(context.Background(), &out)).To(Succeed()) + Expect(informerCache.List(ctx, &out)).To(Succeed()) By("verifying that the returned pods were transformed") for i := 0; i < len(out.Items); i++ { @@ -266,11 +294,11 @@ var _ = Describe("Cache with transformers", func() { } }) - It("should apply default transformer to objects when none is specified", func() { + It("should apply default transformer to objects when none is specified", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service was transformed") Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) @@ -278,7 +306,7 @@ var _ = Describe("Cache with transformers", func() { }) Context("with unstructured objects", func() { - It("should apply transformers to explicitly specified GVKS", func() { + It("should apply transformers to explicitly specified GVKS", func(ctx SpecContext) { By("listing pods") out := unstructured.UnstructuredList{} out.SetGroupVersionKind(schema.GroupVersionKind{ @@ -286,7 +314,7 @@ var _ = Describe("Cache with transformers", func() { Version: "v1", Kind: "PodList", }) - Expect(informerCache.List(context.Background(), &out)).To(Succeed()) + Expect(informerCache.List(ctx, &out)).To(Succeed()) By("verifying that the returned pods were transformed") for i := 0; i < len(out.Items); i++ { @@ -294,7 +322,7 @@ var _ = Describe("Cache with transformers", func() { } }) - It("should apply default transformer to objects when none is specified", func() { + It("should apply default transformer to objects when none is specified", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &unstructured.Unstructured{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -303,7 +331,7 @@ var _ = Describe("Cache with transformers", func() { Kind: "Service", }) svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service was transformed") Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) @@ -311,7 +339,7 @@ var _ = Describe("Cache with transformers", func() { }) Context("with metadata-only objects", func() { - It("should apply transformers to explicitly specified GVKS", func() { + It("should apply transformers to explicitly specified GVKS", func(ctx SpecContext) { By("listing pods") out := metav1.PartialObjectMetadataList{} out.SetGroupVersionKind(schema.GroupVersionKind{ @@ -319,14 +347,14 @@ var _ = Describe("Cache with transformers", func() { Version: "v1", Kind: "PodList", }) - Expect(informerCache.List(context.Background(), &out)).To(Succeed()) + Expect(informerCache.List(ctx, &out)).To(Succeed()) By("verifying that the returned pods were transformed") for i := 0; i < len(out.Items); i++ { Expect(getTransformValue(&out.Items[i])).To(BeIdenticalTo("explicit")) } }) - It("should apply default transformer to objects when none is specified", func() { + It("should apply default transformer to objects when none is specified", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &metav1.PartialObjectMetadata{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -335,7 +363,7 @@ var _ = Describe("Cache with transformers", func() { Kind: "Service", }) svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service was transformed") Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) @@ -347,28 +375,32 @@ var _ = Describe("Cache with selectors", func() { defer GinkgoRecover() var ( informerCache cache.Cache - informerCacheCtx context.Context informerCacheCancel context.CancelFunc ) - BeforeEach(func() { - informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo Expect(cfg).NotTo(BeNil()) cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceOne, cl) + err = ensureNamespace(ctx, testNamespaceOne, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceTwo, cl) + err = ensureNamespace(ctx, testNamespaceTwo, cl) Expect(err).NotTo(HaveOccurred()) for idx, namespace := range []string{testNamespaceOne, testNamespaceTwo} { - _ = createSA("test-sa-"+strconv.Itoa(idx), namespace, cl) - _ = createSvc("test-svc-"+strconv.Itoa(idx), namespace, cl) + _ = createSA(ctx, "test-sa-"+strconv.Itoa(idx), namespace, cl) + _ = createSvc(ctx, "test-svc-"+strconv.Itoa(idx), namespace, cl) } opts := cache.Options{ DefaultFieldSelector: fields.OneTermEqualSelector("metadata.namespace", testNamespaceTwo), ByObject: map[client.Object]cache.ByObject{ - &corev1.ServiceAccount{}: {Field: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne)}, + &corev1.ServiceAccount{}: { + Field: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne), + }, }, } @@ -384,8 +416,7 @@ var _ = Describe("Cache with selectors", func() { Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) }) - AfterEach(func() { - ctx := context.Background() + AfterEach(func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) for idx, namespace := range []string{testNamespaceOne, testNamespaceTwo} { @@ -397,28 +428,181 @@ var _ = Describe("Cache with selectors", func() { informerCacheCancel() }) - It("Should list serviceaccounts and find exactly one in namespace "+testNamespaceOne, func() { + It("Should list serviceaccounts and find exactly one in namespace "+testNamespaceOne, func(ctx SpecContext) { var sas corev1.ServiceAccountList - err := informerCache.List(informerCacheCtx, &sas) + err := informerCache.List(ctx, &sas) Expect(err).NotTo(HaveOccurred()) Expect(sas.Items).To(HaveLen(1)) Expect(sas.Items[0].Namespace).To(Equal(testNamespaceOne)) }) - It("Should list services and find exactly one in namespace "+testNamespaceTwo, func() { + It("Should list services and find exactly one in namespace "+testNamespaceTwo, func(ctx SpecContext) { var svcs corev1.ServiceList - err := informerCache.List(informerCacheCtx, &svcs) + err := informerCache.List(ctx, &svcs) Expect(err).NotTo(HaveOccurred()) Expect(svcs.Items).To(HaveLen(1)) Expect(svcs.Items[0].Namespace).To(Equal(testNamespaceTwo)) }) }) +func CacheTestReaderFailOnMissingInformer(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error), opts cache.Options) { + Describe("Cache test with ReaderFailOnMissingInformer = true", func() { + var ( + informerCache cache.Cache + informerCacheCancel context.CancelFunc + errNotCached *cache.ErrResourceNotCached + ) + + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo + Expect(cfg).NotTo(BeNil()) + By("creating the informer cache") + var err error + informerCache, err = createCacheFunc(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) + }) + + AfterEach(func() { + informerCacheCancel() + }) + + Describe("as a Reader", func() { + Context("with structured objects", func() { + It("should not be able to list objects that haven't been watched previously", func(ctx SpecContext) { + By("listing all services in the cluster") + listObj := &corev1.ServiceList{} + Expect(errors.As(informerCache.List(ctx, listObj), &errNotCached)).To(BeTrue()) + }) + + It("should not be able to get objects that haven't been watched previously", func(ctx SpecContext) { + By("getting the Kubernetes service") + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(errors.As(informerCache.Get(ctx, svcKey, svc), &errNotCached)).To(BeTrue()) + }) + + It("should be able to list objects that are configured to be watched", func(ctx SpecContext) { + By("indicating that we need to watch services") + _, err := informerCache.GetInformer(ctx, &corev1.Service{}) + Expect(err).ToNot(HaveOccurred()) + + By("listing all services in the cluster") + svcList := &corev1.ServiceList{} + Expect(informerCache.List(ctx, svcList)).To(Succeed()) + + By("verifying that the returned service looks reasonable") + Expect(svcList.Items).To(HaveLen(1)) + Expect(svcList.Items[0].Name).To(Equal("kubernetes")) + Expect(svcList.Items[0].Namespace).To(Equal("default")) + }) + + It("should be able to get objects that are configured to be watched", func(ctx SpecContext) { + By("indicating that we need to watch services") + _, err := informerCache.GetInformer(ctx, &corev1.Service{}) + Expect(err).ToNot(HaveOccurred()) + + By("getting the Kubernetes service") + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) + + By("verifying that the returned service looks reasonable") + Expect(svc.Name).To(Equal("kubernetes")) + Expect(svc.Namespace).To(Equal("default")) + }) + }) + }) + }) +} + +func NonBlockingGetTest(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error), opts cache.Options) { + Describe("non-blocking get test", func() { + var ( + informerCache cache.Cache + informerCacheCancel context.CancelFunc + ) + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo + Expect(cfg).NotTo(BeNil()) + + By("creating expected namespaces") + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + err = ensureNode(ctx, testNodeOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceTwo, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(ctx, testNamespaceThree, cl) + Expect(err).NotTo(HaveOccurred()) + + By("creating the informer cache") + opts.NewInformer = func(_ kcache.ListerWatcher, _ runtime.Object, _ time.Duration, _ kcache.Indexers) kcache.SharedIndexInformer { + return &controllertest.FakeInformer{Synced: false} + } + informerCache, err = createCacheFunc(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) + }) + + AfterEach(func() { + By("cleaning up created pods") + informerCacheCancel() + }) + + Describe("as an Informer", func() { + It("should be able to get informer for the object without blocking", func(specCtx SpecContext) { + By("getting a shared index informer for a pod") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "informer-obj", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + } + + ctx, cancel := context.WithTimeout(specCtx, 5*time.Second) + defer cancel() + sii, err := informerCache.GetInformer(ctx, pod, cache.BlockUntilSynced(false)) + Expect(err).NotTo(HaveOccurred()) + Expect(sii).NotTo(BeNil()) + Expect(sii.HasSynced()).To(BeFalse()) + }) + }) + }) +} + func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error), opts cache.Options) { Describe("Cache test", func() { var ( informerCache cache.Cache - informerCacheCtx context.Context informerCacheCancel context.CancelFunc knownPod1 client.Object knownPod2 client.Object @@ -428,28 +612,33 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca knownPod6 client.Object ) - BeforeEach(func() { - informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) + BeforeEach(func(ctx SpecContext) { + var informerCacheCtx context.Context + // Has to be derived from context.Background as it has to stay valid past the + // BeforeEach. + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) //nolint:forbidigo Expect(cfg).NotTo(BeNil()) By("creating three pods") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - err = ensureNode(testNodeOne, cl) + err = ensureNode(ctx, testNodeOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNode(ctx, testNodeTwo, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceOne, cl) + err = ensureNamespace(ctx, testNamespaceOne, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceTwo, cl) + err = ensureNamespace(ctx, testNamespaceTwo, cl) Expect(err).NotTo(HaveOccurred()) - err = ensureNamespace(testNamespaceThree, cl) + err = ensureNamespace(ctx, testNamespaceThree, cl) Expect(err).NotTo(HaveOccurred()) // Includes restart policy since these objects are indexed on this field. - knownPod1 = createPod("test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) - knownPod2 = createPod("test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) - knownPod3 = createPodWithLabels("test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) - knownPod4 = createPodWithLabels("test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) - knownPod5 = createPod("test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) - knownPod6 = createPod("test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod1 = createPod(ctx, "test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) + knownPod2 = createPod(ctx, "test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod3 = createPodWithLabels(ctx, "test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) + knownPod4 = createPodWithLabels(ctx, "test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) + knownPod5 = createPod(ctx, "test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) + knownPod6 = createPod(ctx, "test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) podGVK := schema.GroupVersionKind{ Kind: "Pod", @@ -472,27 +661,27 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca defer GinkgoRecover() Expect(informerCache.Start(ctx)).To(Succeed()) }(informerCacheCtx) - Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informerCache.WaitForCacheSync(ctx)).To(BeTrue()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { By("cleaning up created pods") - deletePod(knownPod1) - deletePod(knownPod2) - deletePod(knownPod3) - deletePod(knownPod4) - deletePod(knownPod5) - deletePod(knownPod6) + deletePod(ctx, knownPod1) + deletePod(ctx, knownPod2) + deletePod(ctx, knownPod3) + deletePod(ctx, knownPod4) + deletePod(ctx, knownPod5) + deletePod(ctx, knownPod6) informerCacheCancel() }) Describe("as a Reader", func() { Context("with structured objects", func() { - It("should be able to list objects that haven't been watched previously", func() { + It("should be able to list objects that haven't been watched previously", func(ctx SpecContext) { By("listing all services in the cluster") listObj := &corev1.ServiceList{} - Expect(informerCache.List(context.Background(), listObj)).To(Succeed()) + Expect(informerCache.List(ctx, listObj)).To(Succeed()) By("verifying that the returned list contains the Kubernetes service") // NB: kubernetes default service is automatically created in testenv. @@ -508,22 +697,22 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(hasKubeService).To(BeTrue()) }) - It("should be able to get objects that haven't been watched previously", func() { + It("should be able to get objects that haven't been watched previously", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service looks reasonable") Expect(svc.Name).To(Equal("kubernetes")) Expect(svc.Namespace).To(Equal("default")) }) - It("should support filtering by labels in a single namespace", func() { + It("should support filtering by labels in a single namespace", func(ctx SpecContext) { By("listing pods with a particular label") // NB: each pod has a "test-label": out := corev1.PodList{} - Expect(informerCache.List(context.Background(), &out, + Expect(informerCache.List(ctx, &out, client.InNamespace(testNamespaceTwo), client.MatchingLabels(map[string]string{"test-label": "test-pod-2"}))).To(Succeed()) @@ -534,16 +723,16 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.Labels["test-label"]).To(Equal("test-pod-2")) }) - It("should support filtering by labels from multiple namespaces", func() { + It("should support filtering by labels from multiple namespaces", func(ctx SpecContext) { By("creating another pod with the same label but different namespace") - anotherPod := createPod("test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) - defer deletePod(anotherPod) + anotherPod := createPod(ctx, "test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) + defer deletePod(ctx, anotherPod) By("listing pods with a particular label") // NB: each pod has a "test-label": out := corev1.PodList{} labels := map[string]string{"test-label": "test-pod-2"} - Expect(informerCache.List(context.Background(), &out, client.MatchingLabels(labels))).To(Succeed()) + Expect(informerCache.List(ctx, &out, client.MatchingLabels(labels))).To(Succeed()) By("verifying multiple pods with the same label in different namespaces are returned") Expect(out.Items).NotTo(BeEmpty()) @@ -554,10 +743,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) if !isPodDisableDeepCopy(opts) { - It("should be able to list objects with GVK populated", func() { + It("should be able to list objects with GVK populated", func(ctx SpecContext) { By("listing pods") out := &corev1.PodList{} - Expect(informerCache.List(context.Background(), out)).To(Succeed()) + Expect(informerCache.List(ctx, out)).To(Succeed()) By("verifying that the returned pods have GVK populated") Expect(out.Items).NotTo(BeEmpty()) @@ -568,10 +757,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should be able to list objects by namespace", func() { + It("should be able to list objects by namespace", func(ctx SpecContext) { By("listing pods in test-namespace-1") listObj := &corev1.PodList{} - Expect(informerCache.List(context.Background(), listObj, + Expect(informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying that the returned pods are in test-namespace-1") @@ -583,11 +772,11 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) if !isPodDisableDeepCopy(opts) { - It("should deep copy the object unless told otherwise", func() { + It("should deep copy the object unless told otherwise", func(ctx SpecContext) { By("retrieving a specific pod from the cache") out := &corev1.Pod{} podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out)).To(Succeed()) By("verifying the retrieved pod is equal to a known pod") Expect(out).To(Equal(knownPod2)) @@ -599,13 +788,13 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(out).NotTo(Equal(knownPod2)) }) } else { - It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func(ctx SpecContext) { By("getting a specific pod from the cache twice") podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} out1 := &corev1.Pod{} - Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out1)).To(Succeed()) out2 := &corev1.Pod{} - Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out2)).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") Expect(out1).To(Equal(out2)) @@ -613,9 +802,9 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("listing pods from the cache twice") outList1 := &corev1.PodList{} - Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) outList2 := &corev1.PodList{} - Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") Expect(outList1.Items).To(HaveLen(len(outList2.Items))) @@ -630,60 +819,81 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should return an error if the object is not found", func() { + It("should return an error if the object is not found", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: testNamespaceOne, Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return an error if getting object in unwatched namespace", func() { + It("should return an error if getting object in unwatched namespace", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "unknown", Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) }) - It("should return an error when context is cancelled", func() { + It("should return an error when context is cancelled", func(specCtx SpecContext) { By("cancelling the context") - informerCacheCancel() + ctx := cancelledCtx(specCtx) By("listing pods in test-namespace-1 with a cancelled context") listObj := &corev1.PodList{} - err := informerCache.List(informerCacheCtx, listObj, client.InNamespace(testNamespaceOne)) + err := informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne)) By("verifying that an error is returned") Expect(err).To(HaveOccurred()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) - It("should set the Limit option and limit number of objects to Limit when List is called", func() { + It("should set the Limit option and limit number of objects to Limit when List is called", func(ctx SpecContext) { opts := &client.ListOptions{Limit: int64(3)} By("verifying that only Limit (3) number of objects are retrieved from the cache") listObj := &corev1.PodList{} - Expect(informerCache.List(context.Background(), listObj, opts)).To(Succeed()) + Expect(informerCache.List(ctx, listObj, opts)).To(Succeed()) Expect(listObj.Items).Should(HaveLen(3)) }) - It("should return a limited result set matching the correct label", func() { + It("should return a limited result set matching the correct label", func(ctx SpecContext) { listObj := &corev1.PodList{} labelOpt := client.MatchingLabels(map[string]string{"common-label": "common"}) limitOpt := client.Limit(1) By("verifying that only Limit (1) number of objects are retrieved from the cache") - Expect(informerCache.List(context.Background(), listObj, labelOpt, limitOpt)).To(Succeed()) + Expect(informerCache.List(ctx, listObj, labelOpt, limitOpt)).To(Succeed()) Expect(listObj.Items).Should(HaveLen(1)) }) + + It("should return an error if pagination is used", func(ctx SpecContext) { + listObj := &corev1.PodList{} + By("verifying that the first list works and returns a sentinel continue") + err := informerCache.List(ctx, listObj) + Expect(err).ToNot(HaveOccurred()) + Expect(listObj.Continue).To(Equal("continue-not-supported")) + + By("verifying that an error is returned") + err = informerCache.List(ctx, listObj, client.Continue(listObj.Continue)) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("continue list option is not supported by the cache")) + }) + + It("should return an error if the continue list options is set", func(ctx SpecContext) { + listObj := &corev1.PodList{} + continueOpt := client.Continue("token") + By("verifying that an error is returned") + err := informerCache.List(ctx, listObj, continueOpt) + Expect(err).To(HaveOccurred()) + }) }) Context("with unstructured objects", func() { - It("should be able to list objects that haven't been watched previously", func() { + It("should be able to list objects that haven't been watched previously", func(ctx SpecContext) { By("listing all services in the cluster") listObj := &unstructured.UnstructuredList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -691,7 +901,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "ServiceList", }) - err := informerCache.List(context.Background(), listObj) + err := informerCache.List(ctx, listObj) Expect(err).To(Succeed()) By("verifying that the returned list contains the Kubernetes service") @@ -707,7 +917,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } Expect(hasKubeService).To(BeTrue()) }) - It("should be able to get objects that haven't been watched previously", func() { + It("should be able to get objects that haven't been watched previously", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &unstructured.Unstructured{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -716,14 +926,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "Service", }) svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service looks reasonable") Expect(svc.GetName()).To(Equal("kubernetes")) Expect(svc.GetNamespace()).To(Equal("default")) }) - It("should support filtering by labels in a single namespace", func() { + It("should support filtering by labels in a single namespace", func(ctx SpecContext) { By("listing pods with a particular label") // NB: each pod has a "test-label": out := unstructured.UnstructuredList{} @@ -732,7 +942,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), &out, + err := informerCache.List(ctx, &out, client.InNamespace(testNamespaceTwo), client.MatchingLabels(map[string]string{"test-label": "test-pod-2"})) Expect(err).To(Succeed()) @@ -744,10 +954,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.GetLabels()["test-label"]).To(Equal("test-pod-2")) }) - It("should support filtering by labels from multiple namespaces", func() { + It("should support filtering by labels from multiple namespaces", func(ctx SpecContext) { By("creating another pod with the same label but different namespace") - anotherPod := createPod("test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) - defer deletePod(anotherPod) + anotherPod := createPod(ctx, "test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) + defer deletePod(ctx, anotherPod) By("listing pods with a particular label") // NB: each pod has a "test-label": @@ -758,7 +968,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "PodList", }) labels := map[string]string{"test-label": "test-pod-2"} - err := informerCache.List(context.Background(), &out, client.MatchingLabels(labels)) + err := informerCache.List(ctx, &out, client.MatchingLabels(labels)) Expect(err).To(Succeed()) By("verifying multiple pods with the same label in different namespaces are returned") @@ -769,7 +979,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to list objects by namespace", func() { + It("should be able to list objects by namespace", func(ctx SpecContext) { By("listing pods in test-namespace-1") listObj := &unstructured.UnstructuredList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -777,7 +987,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), listObj, client.InNamespace(testNamespaceOne)) + err := informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne)) Expect(err).To(Succeed()) By("verifying that the returned pods are in test-namespace-1") @@ -788,64 +998,94 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to restrict cache to a namespace", func() { - By("creating a namespaced cache") - namespacedCache, err := cache.New(cfg, cache.Options{Namespaces: []string{testNamespaceOne}}) - Expect(err).NotTo(HaveOccurred()) + cacheRestrictSubTests := []struct { + nameSuffix string + cacheOpts cache.Options + }{ + { + nameSuffix: "by using the per-gvk setting", + cacheOpts: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Namespaces: map[string]cache.Config{ + testNamespaceOne: {}, + }, + }, + }, + }, + }, + { + nameSuffix: "by using the global DefaultNamespaces setting", + cacheOpts: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceOne: {}, + }, + }, + }, + } - By("running the cache and waiting for it to sync") - go func() { - defer GinkgoRecover() - Expect(namespacedCache.Start(informerCacheCtx)).To(Succeed()) - }() - Expect(namespacedCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + for _, tc := range cacheRestrictSubTests { + It("should be able to restrict cache to a namespace "+tc.nameSuffix, func(ctx SpecContext) { + By("creating a namespaced cache") + namespacedCache, err := cache.New(cfg, tc.cacheOpts) + Expect(err).NotTo(HaveOccurred()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(namespacedCache.Start(ctx)).To(Succeed()) + }() + Expect(namespacedCache.WaitForCacheSync(ctx)).To(BeTrue()) + + By("listing pods in all namespaces") + out := &unstructured.UnstructuredList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + for range 2 { + Expect(namespacedCache.List(ctx, out)).To(Succeed()) + + By("verifying the returned pod is from the watched namespace") + Expect(out.Items).NotTo(BeEmpty()) + Expect(out.Items).Should(HaveLen(2)) + for _, item := range out.Items { + Expect(item.GetNamespace()).To(Equal(testNamespaceOne)) + } + } + By("listing all nodes - should still be able to list a cluster-scoped resource") + nodeList := &unstructured.UnstructuredList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + Expect(namespacedCache.List(ctx, nodeList)).To(Succeed()) - By("listing pods in all namespaces") - out := &unstructured.UnstructuredList{} - out.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "PodList", - }) - Expect(namespacedCache.List(context.Background(), out)).To(Succeed()) + By("verifying the node list is not empty") + Expect(nodeList.Items).NotTo(BeEmpty()) - By("verifying the returned pod is from the watched namespace") - Expect(out.Items).NotTo(BeEmpty()) - Expect(out.Items).Should(HaveLen(2)) - for _, item := range out.Items { - Expect(item.GetNamespace()).To(Equal(testNamespaceOne)) - } - By("listing all nodes - should still be able to list a cluster-scoped resource") - nodeList := &unstructured.UnstructuredList{} - nodeList.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "NodeList", - }) - Expect(namespacedCache.List(context.Background(), nodeList)).To(Succeed()) + By("getting a node - should still be able to get a cluster-scoped resource") + node := &unstructured.Unstructured{} + node.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Node", + }) - By("verifying the node list is not empty") - Expect(nodeList.Items).NotTo(BeEmpty()) + By("verifying that getting the node works with an empty namespace") + key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} + Expect(namespacedCache.Get(ctx, key1, node)).To(Succeed()) - By("getting a node - should still be able to get a cluster-scoped resource") - node := &unstructured.Unstructured{} - node.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Node", + By("verifying that the namespace is ignored when getting a cluster-scoped resource") + key2 := client.ObjectKey{Namespace: "random", Name: testNodeOne} + Expect(namespacedCache.Get(ctx, key2, node)).To(Succeed()) }) - - By("verifying that getting the node works with an empty namespace") - key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key1, node)).To(Succeed()) - - By("verifying that the namespace is ignored when getting a cluster-scoped resource") - key2 := client.ObjectKey{Namespace: "random", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key2, node)).To(Succeed()) - }) + } if !isPodDisableDeepCopy(opts) { - It("should deep copy the object unless told otherwise", func() { + It("should deep copy the object unless told otherwise", func(ctx SpecContext) { By("retrieving a specific pod from the cache") out := &unstructured.Unstructured{} out.SetGroupVersionKind(schema.GroupVersionKind{ @@ -857,7 +1097,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(kscheme.Scheme.Convert(knownPod2, uKnownPod2, nil)).To(Succeed()) podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out)).To(Succeed()) By("verifying the retrieved pod is equal to a known pod") Expect(out).To(Equal(uKnownPod2)) @@ -870,15 +1110,15 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(out).NotTo(Equal(knownPod2)) }) } else { - It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func(ctx SpecContext) { By("getting a specific pod from the cache twice") podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} out1 := &unstructured.Unstructured{} out1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out1)).To(Succeed()) out2 := &unstructured.Unstructured{} out2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out2)).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") Expect(out1).To(Equal(out2)) @@ -887,10 +1127,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("listing pods from the cache twice") outList1 := &unstructured.UnstructuredList{} outList1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) outList2 := &unstructured.UnstructuredList{} outList2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") Expect(outList1.Items).To(HaveLen(len(outList2.Items))) @@ -905,7 +1145,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should return an error if the object is not found", func() { + It("should return an error if the object is not found", func(ctx SpecContext) { By("getting a service that does not exists") svc := &unstructured.Unstructured{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -916,38 +1156,42 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca svcKey := client.ObjectKey{Namespace: testNamespaceOne, Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return an error if getting object in unwatched namespace", func() { + It("should return an error if getting object in unwatched namespace", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "unknown", Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) }) - It("test multinamespaced cache for cluster scoped resources", func() { + It("test multinamespaced cache for cluster scoped resources", func(ctx SpecContext) { By("creating a multinamespaced cache to watch specific namespaces") - multi := cache.MultiNamespacedCacheBuilder([]string{"default", testNamespaceOne}) - m, err := multi(cfg, cache.Options{}) + m, err := cache.New(cfg, cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + "default": {}, + testNamespaceOne: {}, + }, + }) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting it for sync") go func() { defer GinkgoRecover() - Expect(m.Start(informerCacheCtx)).To(Succeed()) + Expect(m.Start(ctx)).To(Succeed()) }() - Expect(m.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(m.WaitForCacheSync(ctx)).To(BeTrue()) By("should be able to fetch cluster scoped resource") node := &corev1.Node{} By("verifying that getting the node works with an empty namespace") key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} - Expect(m.Get(context.Background(), key1, node)).To(Succeed()) + Expect(m.Get(ctx, key1, node)).To(Succeed()) By("verifying if the cluster scoped resources are not duplicated") nodeList := &unstructured.UnstructuredList{} @@ -956,15 +1200,41 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "NodeList", }) - Expect(m.List(context.Background(), nodeList)).To(Succeed()) + Expect(m.List(ctx, nodeList)).To(Succeed()) By("verifying the node list is not empty") Expect(nodeList.Items).NotTo(BeEmpty()) - Expect(len(nodeList.Items)).To(BeEquivalentTo(1)) + Expect(len(nodeList.Items)).To(BeEquivalentTo(2)) + }) + + It("should return an error if pagination is used", func(ctx SpecContext) { + nodeList := &unstructured.UnstructuredList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + By("verifying that the first list works and returns a sentinel continue") + err := informerCache.List(ctx, nodeList) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeList.GetContinue()).To(Equal("continue-not-supported")) + + By("verifying that an error is returned") + err = informerCache.List(ctx, nodeList, client.Continue(nodeList.GetContinue())) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("continue list option is not supported by the cache")) + }) + + It("should return an error if the continue list options is set", func(ctx SpecContext) { + podList := &unstructured.Unstructured{} + continueOpt := client.Continue("token") + By("verifying that an error is returned") + err := informerCache.List(ctx, podList, continueOpt) + Expect(err).To(HaveOccurred()) }) }) Context("with metadata-only objects", func() { - It("should be able to list objects that haven't been watched previously", func() { + It("should be able to list objects that haven't been watched previously", func(ctx SpecContext) { By("listing all services in the cluster") listObj := &metav1.PartialObjectMetadataList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -972,7 +1242,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "ServiceList", }) - err := informerCache.List(context.Background(), listObj) + err := informerCache.List(ctx, listObj) Expect(err).To(Succeed()) By("verifying that the returned list contains the Kubernetes service") @@ -988,7 +1258,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } Expect(hasKubeService).To(BeTrue()) }) - It("should be able to get objects that haven't been watched previously", func() { + It("should be able to get objects that haven't been watched previously", func(ctx SpecContext) { By("getting the Kubernetes service") svc := &metav1.PartialObjectMetadata{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -997,14 +1267,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "Service", }) svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} - Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + Expect(informerCache.Get(ctx, svcKey, svc)).To(Succeed()) By("verifying that the returned service looks reasonable") Expect(svc.GetName()).To(Equal("kubernetes")) Expect(svc.GetNamespace()).To(Equal("default")) }) - It("should support filtering by labels in a single namespace", func() { + It("should support filtering by labels in a single namespace", func(ctx SpecContext) { By("listing pods with a particular label") // NB: each pod has a "test-label": out := metav1.PartialObjectMetadataList{} @@ -1013,7 +1283,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), &out, + err := informerCache.List(ctx, &out, client.InNamespace(testNamespaceTwo), client.MatchingLabels(map[string]string{"test-label": "test-pod-2"})) Expect(err).To(Succeed()) @@ -1025,10 +1295,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.GetLabels()["test-label"]).To(Equal("test-pod-2")) }) - It("should support filtering by labels from multiple namespaces", func() { + It("should support filtering by labels from multiple namespaces", func(ctx SpecContext) { By("creating another pod with the same label but different namespace") - anotherPod := createPod("test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) - defer deletePod(anotherPod) + anotherPod := createPod(ctx, "test-pod-2", testNamespaceOne, corev1.RestartPolicyAlways) + defer deletePod(ctx, anotherPod) By("listing pods with a particular label") // NB: each pod has a "test-label": @@ -1039,7 +1309,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "PodList", }) labels := map[string]string{"test-label": "test-pod-2"} - err := informerCache.List(context.Background(), &out, client.MatchingLabels(labels)) + err := informerCache.List(ctx, &out, client.MatchingLabels(labels)) Expect(err).To(Succeed()) By("verifying multiple pods with the same label in different namespaces are returned") @@ -1050,7 +1320,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to list objects by namespace", func() { + It("should be able to list objects by namespace", func(ctx SpecContext) { By("listing pods in test-namespace-1") listObj := &metav1.PartialObjectMetadataList{} listObj.SetGroupVersionKind(schema.GroupVersionKind{ @@ -1058,7 +1328,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err := informerCache.List(context.Background(), listObj, client.InNamespace(testNamespaceOne)) + err := informerCache.List(ctx, listObj, client.InNamespace(testNamespaceOne)) Expect(err).To(Succeed()) By("verifying that the returned pods are in test-namespace-1") @@ -1069,17 +1339,17 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to restrict cache to a namespace", func() { + It("should be able to restrict cache to a namespace", func(ctx SpecContext) { By("creating a namespaced cache") - namespacedCache, err := cache.New(cfg, cache.Options{Namespaces: []string{testNamespaceOne}}) + namespacedCache, err := cache.New(cfg, cache.Options{DefaultNamespaces: map[string]cache.Config{testNamespaceOne: {}}}) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(namespacedCache.Start(informerCacheCtx)).To(Succeed()) + Expect(namespacedCache.Start(ctx)).To(Succeed()) }() - Expect(namespacedCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(namespacedCache.WaitForCacheSync(ctx)).To(BeTrue()) By("listing pods in all namespaces") out := &metav1.PartialObjectMetadataList{} @@ -1088,7 +1358,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - Expect(namespacedCache.List(context.Background(), out)).To(Succeed()) + Expect(namespacedCache.List(ctx, out)).To(Succeed()) By("verifying the returned pod is from the watched namespace") Expect(out.Items).NotTo(BeEmpty()) @@ -1103,7 +1373,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "NodeList", }) - Expect(namespacedCache.List(context.Background(), nodeList)).To(Succeed()) + Expect(namespacedCache.List(ctx, nodeList)).To(Succeed()) By("verifying the node list is not empty") Expect(nodeList.Items).NotTo(BeEmpty()) @@ -1118,15 +1388,84 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("verifying that getting the node works with an empty namespace") key1 := client.ObjectKey{Namespace: "", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key1, node)).To(Succeed()) + Expect(namespacedCache.Get(ctx, key1, node)).To(Succeed()) By("verifying that the namespace is ignored when getting a cluster-scoped resource") key2 := client.ObjectKey{Namespace: "random", Name: testNodeOne} - Expect(namespacedCache.Get(context.Background(), key2, node)).To(Succeed()) + Expect(namespacedCache.Get(ctx, key2, node)).To(Succeed()) + }) + + It("should be able to restrict cache to a namespace for namespaced object and to given selectors for non namespaced object", func(ctx SpecContext) { + By("creating a namespaced cache") + namespacedCache, err := cache.New(cfg, cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespaceOne: {}}, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Node{}: { + Label: labels.SelectorFromSet(labels.Set{"name": testNodeTwo}), + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(namespacedCache.Start(ctx)).To(Succeed()) + }() + Expect(namespacedCache.WaitForCacheSync(ctx)).To(BeTrue()) + + By("listing pods in all namespaces") + out := &metav1.PartialObjectMetadataList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + Expect(namespacedCache.List(ctx, out)).To(Succeed()) + + By("verifying the returned pod is from the watched namespace") + Expect(out.Items).NotTo(BeEmpty()) + Expect(out.Items).Should(HaveLen(2)) + for _, item := range out.Items { + Expect(item.Namespace).To(Equal(testNamespaceOne)) + } + By("listing all nodes - should still be able to list a cluster-scoped resource") + nodeList := &metav1.PartialObjectMetadataList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + Expect(namespacedCache.List(ctx, nodeList)).To(Succeed()) + + By("verifying the node list is not empty") + Expect(nodeList.Items).NotTo(BeEmpty()) + + By("getting a node - should still be able to get a cluster-scoped resource") + node := &metav1.PartialObjectMetadata{} + node.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Node", + }) + + By("verifying that getting the node works with an empty namespace") + key1 := client.ObjectKey{Namespace: "", Name: testNodeTwo} + Expect(namespacedCache.Get(ctx, key1, node)).To(Succeed()) + + By("verifying that the namespace is ignored when getting a cluster-scoped resource") + key2 := client.ObjectKey{Namespace: "random", Name: testNodeTwo} + Expect(namespacedCache.Get(ctx, key2, node)).To(Succeed()) + + By("verifying that an error is returned for node with not matching label") + key3 := client.ObjectKey{Namespace: "", Name: testNodeOne} + err = namespacedCache.Get(ctx, key3, node) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) if !isPodDisableDeepCopy(opts) { - It("should deep copy the object unless told otherwise", func() { + It("should deep copy the object unless told otherwise", func(ctx SpecContext) { By("retrieving a specific pod from the cache") out := &metav1.PartialObjectMetadata{} out.SetGroupVersionKind(schema.GroupVersionKind{ @@ -1143,7 +1482,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out)).To(Succeed()) By("verifying the retrieved pod is equal to a known pod") Expect(out).To(Equal(uKnownPod2)) @@ -1155,15 +1494,15 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(out).NotTo(Equal(knownPod2)) }) } else { - It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func(ctx SpecContext) { By("getting a specific pod from the cache twice") podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} out1 := &metav1.PartialObjectMetadata{} out1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out1)).To(Succeed()) out2 := &metav1.PartialObjectMetadata{} out2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) - Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + Expect(informerCache.Get(ctx, podKey, out2)).To(Succeed()) By("verifying the pods have the same pointer addresses") By("verifying the pointer fields in pod have the same addresses") @@ -1173,10 +1512,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("listing pods from the cache twice") outList1 := &metav1.PartialObjectMetadataList{} outList1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) outList2 := &metav1.PartialObjectMetadataList{} outList2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) - Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + Expect(informerCache.List(ctx, outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) By("verifying the pointer fields in pod have the same addresses") Expect(outList1.Items).To(HaveLen(len(outList2.Items))) @@ -1191,7 +1530,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) } - It("should return an error if the object is not found", func() { + It("should return an error if the object is not found", func(ctx SpecContext) { By("getting a service that does not exists") svc := &metav1.PartialObjectMetadata{} svc.SetGroupVersionKind(schema.GroupVersionKind{ @@ -1202,47 +1541,57 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca svcKey := client.ObjectKey{Namespace: testNamespaceOne, Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return an error if getting object in unwatched namespace", func() { + It("should return an error if getting object in unwatched namespace", func(ctx SpecContext) { By("getting a service that does not exists") svc := &corev1.Service{} svcKey := client.ObjectKey{Namespace: "unknown", Name: "unknown"} By("verifying that an error is returned") - err := informerCache.Get(context.Background(), svcKey, svc) + err := informerCache.Get(ctx, svcKey, svc) Expect(err).To(HaveOccurred()) }) + + It("should return an error if pagination is used", func(ctx SpecContext) { + nodeList := &metav1.PartialObjectMetadataList{} + nodeList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "NodeList", + }) + By("verifying that the first list works and returns a sentinel continue") + err := informerCache.List(ctx, nodeList) + Expect(err).ToNot(HaveOccurred()) + Expect(nodeList.GetContinue()).To(Equal("continue-not-supported")) + + By("verifying that an error is returned") + err = informerCache.List(ctx, nodeList, client.Continue(nodeList.GetContinue())) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("continue list option is not supported by the cache")) + }) }) type selectorsTestCase struct { - fieldSelectors map[string]string - labelSelectors map[string]string - expectedPods []string + options cache.Options + expectedPods []string } - DescribeTable(" and cache with selectors", func(tc selectorsTestCase) { + DescribeTable(" and cache with selectors", func(ctx SpecContext, tc selectorsTestCase) { By("creating the cache") - informer, err := cache.New(cfg, cache.Options{ - ByObject: map[client.Object]cache.ByObject{ - &corev1.Pod{}: { - Label: labels.Set(tc.labelSelectors).AsSelector(), - Field: fields.Set(tc.fieldSelectors).AsSelector(), - }, - }, - }) + informer, err := cache.New(cfg, tc.options) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("Checking with structured") obtainedStructuredPodList := corev1.PodList{} - Expect(informer.List(context.Background(), &obtainedStructuredPodList)).To(Succeed()) + Expect(informer.List(ctx, &obtainedStructuredPodList)).To(Succeed()) Expect(obtainedStructuredPodList.Items).Should(WithTransform(func(pods []corev1.Pod) []string { obtainedPodNames := []string{} for _, pod := range pods { @@ -1250,6 +1599,9 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return obtainedPodNames }, ConsistOf(tc.expectedPods))) + for _, pod := range obtainedStructuredPodList.Items { + Expect(informer.Get(ctx, client.ObjectKeyFromObject(&pod), &pod)).To(Succeed()) + } By("Checking with unstructured") obtainedUnstructuredPodList := unstructured.UnstructuredList{} @@ -1258,7 +1610,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err = informer.List(context.Background(), &obtainedUnstructuredPodList) + err = informer.List(ctx, &obtainedUnstructuredPodList) Expect(err).To(Succeed()) Expect(obtainedUnstructuredPodList.Items).Should(WithTransform(func(pods []unstructured.Unstructured) []string { obtainedPodNames := []string{} @@ -1267,6 +1619,9 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return obtainedPodNames }, ConsistOf(tc.expectedPods))) + for _, pod := range obtainedUnstructuredPodList.Items { + Expect(informer.Get(ctx, client.ObjectKeyFromObject(&pod), &pod)).To(Succeed()) + } By("Checking with metadata") obtainedMetadataPodList := metav1.PartialObjectMetadataList{} @@ -1275,7 +1630,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err = informer.List(context.Background(), &obtainedMetadataPodList) + err = informer.List(ctx, &obtainedMetadataPodList) Expect(err).To(Succeed()) Expect(obtainedMetadataPodList.Items).Should(WithTransform(func(pods []metav1.PartialObjectMetadata) []string { obtainedPodNames := []string{} @@ -1284,46 +1639,300 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return obtainedPodNames }, ConsistOf(tc.expectedPods))) + for _, pod := range obtainedMetadataPodList.Items { + Expect(informer.Get(ctx, client.ObjectKeyFromObject(&pod), &pod)).To(Succeed()) + } }, Entry("when selectors are empty it has to inform about all the pods", selectorsTestCase{ - fieldSelectors: map[string]string{}, - labelSelectors: map[string]string{}, - expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, + expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, }), - Entry("when field matches one pod it has to inform about it", selectorsTestCase{ - fieldSelectors: map[string]string{"metadata.name": "test-pod-2"}, - expectedPods: []string{"test-pod-2"}, + Entry("type-level field selector matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Field: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + })}, + }}, + expectedPods: []string{"test-pod-2"}, }), - Entry("when field matches multiple pods it has to inform about all of them", selectorsTestCase{ - fieldSelectors: map[string]string{"metadata.namespace": testNamespaceTwo}, - expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-6"}, + Entry("global field selector matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + }), + }, + expectedPods: []string{"test-pod-2"}, + }), + Entry("type-level field selectors matches multiple pods", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Field: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": testNamespaceTwo, + })}, + }}, + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-6"}, }), - Entry("when label matches one pod it has to inform about it", selectorsTestCase{ - labelSelectors: map[string]string{"test-label": "test-pod-4"}, - expectedPods: []string{"test-pod-4"}, + Entry("global field selectors matches multiple pods", selectorsTestCase{ + options: cache.Options{ + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": testNamespaceTwo, + }), + }, + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-6"}, }), - Entry("when label matches multiple pods it has to inform about all of them", selectorsTestCase{ - labelSelectors: map[string]string{"common-label": "common"}, - expectedPods: []string{"test-pod-3", "test-pod-4"}, + Entry("type-level label selector matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-4", + })}, + }}, + expectedPods: []string{"test-pod-4"}, }), - Entry("when label and field matches one pod it has to inform about about it", selectorsTestCase{ - labelSelectors: map[string]string{"common-label": "common"}, - fieldSelectors: map[string]string{"metadata.namespace": testNamespaceTwo}, - expectedPods: []string{"test-pod-3"}, + Entry("namespaces configured, type-level label selector matches everything, overrides global selector", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespaceOne: {}}, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.Everything()}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"does-not": "match-anything"}), + }, + expectedPods: []string{"test-pod-1", "test-pod-5"}, + }), + Entry("namespaces configured, global selector is used", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{testNamespaceTwo: {}}, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"common-label": "common"}), + }, + expectedPods: []string{"test-pod-3"}, + }), + Entry("global label selector matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-4", + }), + }, + expectedPods: []string{"test-pod-4"}, + }), + Entry("type-level label selector matches multiple pods", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.SelectorFromSet(map[string]string{ + "common-label": "common", + })}, + }}, + expectedPods: []string{"test-pod-3", "test-pod-4"}, + }), + Entry("global label selector matches multiple pods", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{ + "common-label": "common", + }), + }, + expectedPods: []string{"test-pod-3", "test-pod-4"}, + }), + Entry("type-level label and field selector, matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Label: labels.SelectorFromSet(map[string]string{"common-label": "common"}), + Field: fields.SelectorFromSet(map[string]string{"metadata.namespace": testNamespaceTwo}), + }, + }}, + expectedPods: []string{"test-pod-3"}, + }), + Entry("global label and field selector, matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"common-label": "common"}), + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{"metadata.namespace": testNamespaceTwo}), + }, + expectedPods: []string{"test-pod-3"}, + }), + Entry("type-level label selector does not match, no results", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Label: labels.SelectorFromSet(map[string]string{ + "new-label": "new", + })}, + }}, + expectedPods: []string{}, + }), + Entry("global label selector does not match, no results", selectorsTestCase{ + options: cache.Options{ + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{ + "new-label": "new", + }), + }, + expectedPods: []string{}, + }), + Entry("type-level field selector does not match, no results", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Field: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": "new", + })}, + }}, + expectedPods: []string{}, + }), + Entry("global field selector does not match, no results", selectorsTestCase{ + options: cache.Options{ + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.namespace": "new", + }), + }, + expectedPods: []string{}, + }), + Entry("type-level field selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + }), + }, + }}, + }}, + expectedPods: []string{"test-pod-2"}, + }), + Entry("type-level field selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-doesn-exist", + }), + }, + }}, + }}, + expectedPods: []string{}, + }), + Entry("global field selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-2", + }), + }, + }, + }, + expectedPods: []string{"test-pod-2"}, + }), + Entry("global field selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": "test-pod-doesn-exist", + }), + }, + }, + }, + expectedPods: []string{}, + }), + Entry("type-level label selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-2", + }), + }, + }}, + }}, + expectedPods: []string{"test-pod-2"}, + }), + Entry("type-level label selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: {Namespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-doesn-exist", + }), + }, + }}, + }}, + expectedPods: []string{}, + }), + Entry("global label selector on namespace matches one pod", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-2", + }), + }, + }, + }, + expectedPods: []string{"test-pod-2"}, + }), + Entry("global label selector on namespace doesn't match", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + testNamespaceTwo: { + LabelSelector: labels.SelectorFromSet(map[string]string{ + "test-label": "test-pod-doesn-exist", + }), + }, + }, + }, + expectedPods: []string{}, + }), + Entry("Only NamespaceAll in DefaultNamespaces returns all pods", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + }, + }, + expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, + }), + Entry("Only NamespaceAll in ByObject.Namespaces returns all pods", selectorsTestCase{ + options: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Namespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + }, + }, + }, + }, + expectedPods: []string{"test-pod-1", "test-pod-2", "test-pod-3", "test-pod-4", "test-pod-5", "test-pod-6"}, }), - Entry("when label does not match it does not has to inform", selectorsTestCase{ - labelSelectors: map[string]string{"new-label": "new"}, - expectedPods: []string{}, + Entry("NamespaceAll in DefaultNamespaces creates a cache for all Namespaces that are not in DefaultNamespaces", selectorsTestCase{ + options: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + testNamespaceOne: { + // labels.Nothing when serialized matches everything, so we have to construct our own "match nothing" selector + LabelSelector: labels.SelectorFromSet(labels.Set{"no-present": "not-present"})}, + }, + }, + // All pods that are not in NamespaceOne + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-4", "test-pod-6"}, }), - Entry("when field does not match it does not has to inform", selectorsTestCase{ - fieldSelectors: map[string]string{"metadata.namespace": "new"}, - expectedPods: []string{}, + Entry("NamespaceAll in ByObject.Namespaces creates a cache for all Namespaces that are not in ByObject.Namespaces", selectorsTestCase{ + options: cache.Options{ + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Namespaces: map[string]cache.Config{ + metav1.NamespaceAll: {}, + testNamespaceOne: { + // labels.Nothing when serialized matches everything, so we have to construct our own "match nothing" selector + LabelSelector: labels.SelectorFromSet(labels.Set{"no-present": "not-present"})}, + }, + }, + }, + }, + // All pods that are not in NamespaceOne + expectedPods: []string{"test-pod-2", "test-pod-3", "test-pod-4", "test-pod-6"}, }), ) }) Describe("as an Informer", func() { + It("should error when starting the cache a second time", func(ctx SpecContext) { + err := informerCache.Start(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("informer already started")) + }) + Context("with structured objects", func() { - It("should be able to get informer for the object", func() { + It("should be able to get informer for the object", func(ctx SpecContext) { By("getting a shared index informer for a pod") pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -1339,7 +1948,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }, }, } - sii, err := informerCache.GetInformer(context.TODO(), pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1354,16 +1963,52 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("adding an object") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) }) - It("should be able to get an informer by group/version/kind", func() { + It("should be able to stop and restart informers", func(ctx SpecContext) { + By("getting a shared index informer for a pod") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "informer-obj", + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + } + sii, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii).NotTo(BeNil()) + Expect(sii.HasSynced()).To(BeTrue()) + + By("removing the existing informer") + Expect(informerCache.RemoveInformer(ctx, pod)).To(Succeed()) + Eventually(sii.IsStopped).WithTimeout(5 * time.Second).Should(BeTrue()) + + By("recreating the informer") + + sii2, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii2).NotTo(BeNil()) + Expect(sii2.HasSynced()).To(BeTrue()) + + By("validating the two informers are in different states") + Expect(sii.IsStopped()).To(BeTrue()) + Expect(sii2.IsStopped()).To(BeFalse()) + }) + It("should be able to get an informer by group/version/kind", func(ctx SpecContext) { By("getting an shared index informer for gvk = core/v1/pod") gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - sii, err := informerCache.GetInformerForKind(context.TODO(), gvk) + sii, err := informerCache.GetInformerForKind(ctx, gvk) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1392,13 +2037,13 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }, }, } - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) }) - It("should be able to index an object field then retrieve objects by that field", func() { + It("should be able to index an object field then retrieve objects by that field", func(ctx SpecContext) { By("creating the cache") informer, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1408,18 +2053,18 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca indexFunc := func(obj client.Object) []string { return []string{string(obj.(*corev1.Pod).Spec.RestartPolicy)} } - Expect(informer.IndexField(context.TODO(), pod, "spec.restartPolicy", indexFunc)).To(Succeed()) + Expect(informer.IndexField(ctx, pod, "spec.restartPolicy", indexFunc)).To(Succeed()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("listing Pods with restartPolicyOnFailure") listObj := &corev1.PodList{} - Expect(informer.List(context.Background(), listObj, + Expect(informer.List(ctx, listObj, client.MatchingFields{"spec.restartPolicy": "OnFailure"})).To(Succeed()) By("verifying that the returned pods have correct restart policy") Expect(listObj.Items).NotTo(BeEmpty()) @@ -1428,9 +2073,10 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.Name).To(Equal("test-pod-3")) }) - It("should allow for get informer to be cancelled", func() { + It("should allow for get informer to be cancelled", func(specCtx SpecContext) { By("creating a context and cancelling it") - informerCacheCancel() + ctx, cancel := context.WithCancel(specCtx) + cancel() By("getting a shared index informer for a pod with a cancelled context") pod := &corev1.Pod{ @@ -1447,48 +2093,49 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }, }, } - sii, err := informerCache.GetInformer(informerCacheCtx, pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).To(HaveOccurred()) Expect(sii).To(BeNil()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) - It("should allow getting an informer by group/version/kind to be cancelled", func() { + It("should allow getting an informer by group/version/kind to be cancelled", func(specCtx SpecContext) { By("creating a context and cancelling it") - informerCacheCancel() + ctx, cancel := context.WithCancel(specCtx) + cancel() By("getting an shared index informer for gvk = core/v1/pod with a cancelled context") gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} - sii, err := informerCache.GetInformerForKind(informerCacheCtx, gvk) + sii, err := informerCache.GetInformerForKind(ctx, gvk) Expect(err).To(HaveOccurred()) Expect(sii).To(BeNil()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) - It("should be able not to change indexer values after indexing cluster-scope objects", func() { + It("should be able not to change indexer values after indexing cluster-scope objects", func(ctx SpecContext) { By("creating the cache") informer, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) By("indexing the Namespace objects with fixed values before starting") - pod := &corev1.Namespace{} + ns := &corev1.Namespace{} indexerValues := []string{"a", "b", "c"} fieldName := "fixedValues" indexFunc := func(obj client.Object) []string { return indexerValues } - Expect(informer.IndexField(context.TODO(), pod, fieldName, indexFunc)).To(Succeed()) + Expect(informer.IndexField(ctx, ns, fieldName, indexFunc)).To(Succeed()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("listing Namespaces with fixed indexer") listObj := &corev1.NamespaceList{} - Expect(informer.List(context.Background(), listObj, + Expect(informer.List(ctx, listObj, client.MatchingFields{fieldName: "a"})).To(Succeed()) Expect(listObj.Items).NotTo(BeZero()) @@ -1498,9 +2145,54 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(indexerValues[1]).To(Equal("b")) Expect(indexerValues[2]).To(Equal("c")) }) + + It("should be able to matching fields with multiple indexes", func(ctx SpecContext) { + By("creating the cache") + informer, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + + pod := &corev1.Pod{} + By("indexing pods with label before starting") + fieldName1 := "indexByLabel" + indexFunc1 := func(obj client.Object) []string { + return []string{obj.(*corev1.Pod).Labels["common-label"]} + } + Expect(informer.IndexField(ctx, pod, fieldName1, indexFunc1)).To(Succeed()) + By("indexing pods with restart policy before starting") + fieldName2 := "indexByPolicy" + indexFunc2 := func(obj client.Object) []string { + return []string{string(obj.(*corev1.Pod).Spec.RestartPolicy)} + } + Expect(informer.IndexField(ctx, pod, fieldName2, indexFunc2)).To(Succeed()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(informer.Start(ctx)).To(Succeed()) + }() + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) + + By("listing pods with label index") + listObj := &corev1.PodList{} + Expect(informer.List(ctx, listObj, + client.MatchingFields{fieldName1: "common"})).To(Succeed()) + Expect(listObj.Items).To(HaveLen(2)) + + By("listing pods with restart policy index") + listObj = &corev1.PodList{} + Expect(informer.List(ctx, listObj, + client.MatchingFields{fieldName2: string(corev1.RestartPolicyNever)})).To(Succeed()) + Expect(listObj.Items).To(HaveLen(3)) + + By("listing pods with both fixed indexers 1 and 2") + listObj = &corev1.PodList{} + Expect(informer.List(ctx, listObj, + client.MatchingFields{fieldName1: "common", fieldName2: string(corev1.RestartPolicyNever)})).To(Succeed()) + Expect(listObj.Items).To(HaveLen(1)) + }) }) Context("with unstructured objects", func() { - It("should be able to get informer for the object", func() { + It("should be able to get informer for the object", func(ctx SpecContext) { By("getting a shared index informer for a pod") pod := &unstructured.Unstructured{ @@ -1522,7 +2214,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "Pod", }) - sii, err := informerCache.GetInformer(context.TODO(), pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1537,14 +2229,55 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("adding an object") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) }) - It("should be able to index an object field then retrieve objects by that field", func() { + It("should be able to stop and restart informers", func(ctx SpecContext) { + By("getting a shared index informer for a pod") + pod := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "containers": []map[string]interface{}{ + { + "name": "nginx", + "image": "nginx", + }, + }, + }, + }, + } + pod.SetName("informer-obj2") + pod.SetNamespace("default") + pod.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + }) + sii, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii).NotTo(BeNil()) + Expect(sii.HasSynced()).To(BeTrue()) + + By("removing the existing informer") + Expect(informerCache.RemoveInformer(ctx, pod)).To(Succeed()) + Eventually(sii.IsStopped).WithTimeout(5 * time.Second).Should(BeTrue()) + + By("recreating the informer") + sii2, err := informerCache.GetInformer(ctx, pod) + Expect(err).NotTo(HaveOccurred()) + Expect(sii2).NotTo(BeNil()) + Expect(sii2.HasSynced()).To(BeTrue()) + + By("validating the two informers are in different states") + Expect(sii.IsStopped()).To(BeTrue()) + Expect(sii2.IsStopped()).To(BeFalse()) + }) + + It("should be able to index an object field then retrieve objects by that field", func(ctx SpecContext) { By("creating the cache") informer, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1567,14 +2300,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } return []string{fmt.Sprintf("%v", m["restartPolicy"])} } - Expect(informer.IndexField(context.TODO(), pod, "spec.restartPolicy", indexFunc)).To(Succeed()) + Expect(informer.IndexField(ctx, pod, "spec.restartPolicy", indexFunc)).To(Succeed()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("listing Pods with restartPolicyOnFailure") listObj := &unstructured.UnstructuredList{} @@ -1583,7 +2316,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "PodList", }) - err = informer.List(context.Background(), listObj, + err = informer.List(ctx, listObj, client.MatchingFields{"spec.restartPolicy": "OnFailure"}) Expect(err).To(Succeed()) @@ -1594,9 +2327,9 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(actual.GetName()).To(Equal("test-pod-3")) }) - It("should allow for get informer to be cancelled", func() { + It("should allow for get informer to be cancelled", func(specCtx SpecContext) { By("cancelling the context") - informerCacheCancel() + ctx := cancelledCtx(specCtx) By("getting a shared index informer for a pod with a cancelled context") pod := &unstructured.Unstructured{} @@ -1607,14 +2340,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "Pod", }) - sii, err := informerCache.GetInformer(informerCacheCtx, pod) + sii, err := informerCache.GetInformer(ctx, pod) Expect(err).To(HaveOccurred()) Expect(sii).To(BeNil()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) }) Context("with metadata-only objects", func() { - It("should be able to get informer for the object", func() { + It("should be able to get informer for the object", func(ctx SpecContext) { By("getting a shared index informer for a pod") pod := &corev1.Pod{ @@ -1640,7 +2373,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "Pod", }) - sii, err := informerCache.GetInformer(context.TODO(), podMeta) + sii, err := informerCache.GetInformer(ctx, podMeta) Expect(err).NotTo(HaveOccurred()) Expect(sii).NotTo(BeNil()) Expect(sii.HasSynced()).To(BeTrue()) @@ -1655,8 +2388,8 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("adding an object") cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(cl.Create(context.Background(), pod)).To(Succeed()) - defer deletePod(pod) + Expect(cl.Create(ctx, pod)).To(Succeed()) + defer deletePod(ctx, pod) // re-copy the result in so that we can match on it properly pod.ObjectMeta.DeepCopyInto(&podMeta.ObjectMeta) @@ -1664,7 +2397,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Eventually(out).Should(Receive(Equal(podMeta))) }) - It("should be able to index an object field then retrieve objects by that field", func() { + It("should be able to index an object field then retrieve objects by that field", func(ctx SpecContext) { By("creating the cache") informer, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1680,14 +2413,14 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca metadata := obj.(*metav1.PartialObjectMetadata) return []string{metadata.Labels["test-label"]} } - Expect(informer.IndexField(context.TODO(), pod, "metadata.labels.test-label", indexFunc)).To(Succeed()) + Expect(informer.IndexField(ctx, pod, "metadata.labels.test-label", indexFunc)).To(Succeed()) By("running the cache and waiting for it to sync") go func() { defer GinkgoRecover() - Expect(informer.Start(informerCacheCtx)).To(Succeed()) + Expect(informer.Start(ctx)).To(Succeed()) }() - Expect(informer.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + Expect(informer.WaitForCacheSync(ctx)).To(BeTrue()) By("listing Pods with restartPolicyOnFailure") listObj := &metav1.PartialObjectMetadataList{} @@ -1697,7 +2430,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Kind: "PodList", } listObj.SetGroupVersionKind(gvk) - err = informer.List(context.Background(), listObj, + err = informer.List(ctx, listObj, client.MatchingFields{"metadata.labels.test-label": "test-pod-3"}) Expect(err).To(Succeed()) @@ -1718,10 +2451,9 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca })) }) - It("should allow for get informer to be cancelled", func() { + It("should allow for get informer to be cancelled", func(specContext SpecContext) { By("creating a context and cancelling it") - ctx, cancel := context.WithCancel(context.Background()) - cancel() + ctx := cancelledCtx(specContext) By("getting a shared index informer for a pod with a cancelled context") pod := &metav1.PartialObjectMetadata{} @@ -1739,34 +2471,69 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) }) }) - Describe("use UnsafeDisableDeepCopy list options", func() { - It("should be able to change object in informer cache", func() { - By("listing pods") - out := corev1.PodList{} - Expect(informerCache.List(context.Background(), &out, client.UnsafeDisableDeepCopy)).To(Succeed()) - for _, item := range out.Items { - if strings.Compare(item.Name, "test-pod-3") == 0 { // test-pod-3 has labels - item.Labels["UnsafeDisableDeepCopy"] = "true" - break + Context("using UnsafeDisableDeepCopy", func() { + Describe("with ListOptions", func() { + It("should be able to change object in informer cache", func(ctx SpecContext) { + By("listing pods") + out := corev1.PodList{} + Expect(informerCache.List(ctx, &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + for _, item := range out.Items { + if strings.Compare(item.Name, "test-pod-3") == 0 { // test-pod-3 has labels + item.Labels["UnsafeDisableDeepCopy"] = "true" + break + } } - } - By("verifying that the returned pods were changed") - out2 := corev1.PodList{} - Expect(informerCache.List(context.Background(), &out, client.UnsafeDisableDeepCopy)).To(Succeed()) - for _, item := range out2.Items { - if strings.Compare(item.Name, "test-pod-3") == 0 { - Expect(item.Labels["UnsafeDisableDeepCopy"]).To(Equal("true")) - break + By("verifying that the returned pods were changed") + out2 := corev1.PodList{} + Expect(informerCache.List(ctx, &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + for _, item := range out2.Items { + if strings.Compare(item.Name, "test-pod-3") == 0 { + Expect(item.Labels["UnsafeDisableDeepCopy"]).To(Equal("true")) + break + } } - } + }) + }) + Describe("with GetOptions", func() { + It("should be able to change object in informer cache", func(ctx SpecContext) { + out := corev1.Pod{} + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + Expect(informerCache.Get(ctx, podKey, &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + + out.Labels["UnsafeDisableDeepCopy"] = "true" + + By("verifying that the returned pod was changed") + out2 := corev1.Pod{} + Expect(informerCache.Get(ctx, podKey, &out2, client.UnsafeDisableDeepCopy)).To(Succeed()) + Expect(out2.Labels["UnsafeDisableDeepCopy"]).To(Equal("true")) + }) }) }) }) } +var _ = Describe("TransformStripManagedFields", func() { + It("should strip managed fields from an object", func() { + obj := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "foo", + }}, + }} + transformed, err := cache.TransformStripManagedFields()(obj) + Expect(err).NotTo(HaveOccurred()) + Expect(transformed).To(Equal(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{}})) + }) + + It("should not trip over an unexpected object", func() { + transformed, err := cache.TransformStripManagedFields()("foo") + Expect(err).NotTo(HaveOccurred()) + Expect(transformed).To(Equal("foo")) + }) +}) + // ensureNamespace installs namespace of a given name if not exists. -func ensureNamespace(namespace string, client client.Client) error { +func ensureNamespace(ctx context.Context, namespace string, client client.Client) error { ns := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: namespace, @@ -1776,31 +2543,31 @@ func ensureNamespace(namespace string, client client.Client) error { APIVersion: "v1", }, } - err := client.Create(context.TODO(), &ns) + err := client.Create(ctx, &ns) if apierrors.IsAlreadyExists(err) { return nil } return err } -func ensureNode(name string, client client.Client) error { +func ensureNode(ctx context.Context, name string, client client.Client) error { node := corev1.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + Labels: map[string]string{"name": name}, }, TypeMeta: metav1.TypeMeta{ Kind: "Node", APIVersion: "v1", }, } - err := client.Create(context.TODO(), &node) + err := client.Create(ctx, &node) if apierrors.IsAlreadyExists(err) { return nil } return err } -//nolint:interfacer func isKubeService(svc metav1.Object) bool { // grumble grumble linters grumble grumble return svc.GetNamespace() == "default" && svc.GetName() == "kubernetes" @@ -1810,8 +2577,14 @@ func isPodDisableDeepCopy(opts cache.Options) bool { if opts.ByObject[&corev1.Pod{}].UnsafeDisableDeepCopy != nil { return *opts.ByObject[&corev1.Pod{}].UnsafeDisableDeepCopy } - if opts.UnsafeDisableDeepCopy != nil { - return *opts.UnsafeDisableDeepCopy + if opts.DefaultUnsafeDisableDeepCopy != nil { + return *opts.DefaultUnsafeDisableDeepCopy } return false } + +func cancelledCtx(ctx context.Context) context.Context { + cancelCtx, cancel := context.WithCancel(ctx) + cancel() + return cancelCtx +} diff --git a/pkg/cache/defaulting_test.go b/pkg/cache/defaulting_test.go new file mode 100644 index 0000000000..d9d0dcceb3 --- /dev/null +++ b/pkg/cache/defaulting_test.go @@ -0,0 +1,498 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "reflect" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + fuzz "github.com/google/gofuzz" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestDefaultOpts(t *testing.T) { + t.Parallel() + + pod := &corev1.Pod{} + + compare := func(a, b any) string { + return cmp.Diff(a, b, + cmpopts.IgnoreUnexported(Options{}), + cmpopts.IgnoreFields(Options{}, "HTTPClient", "Scheme", "Mapper", "SyncPeriod"), + cmp.Comparer(func(a, b fields.Selector) bool { + if (a != nil) != (b != nil) { + return false + } + if a == nil { + return true + } + return a.String() == b.String() + }), + ) + } + testCases := []struct { + name string + in Options + + verification func(Options) string + }{ + { + name: "ByObject.Namespaces gets defaulted from ByObject", + in: Options{ + ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "default": {}, + }, + Label: labels.SelectorFromSet(map[string]string{"from": "by-object"}), + }}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "by-object"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces gets defaulted from DefaultNamespaces", + in: Options{ + ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "default": {}, + }, + }}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces gets defaulted from DefaultLabelSelector", + in: Options{ + ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "default": {}, + }, + }}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces gets defaulted from DefaultNamespaces", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + } + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Namespaces doesn't get defaulted when its empty", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {Namespaces: map[string]Config{}}}, + DefaultNamespaces: map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-namespaces"})}, + }, + }, + + verification: func(o Options) string { + expected := map[string]Config{} + return cmp.Diff(expected, o.ByObject[pod].Namespaces) + }, + }, + { + name: "ByObject.Labels gets defaulted from DefautLabelSelector", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}) + return cmp.Diff(expected, o.ByObject[pod].Label) + }, + }, + { + name: "ByObject.Labels doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {Label: labels.SelectorFromSet(map[string]string{"from": "by-object"})}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := labels.SelectorFromSet(map[string]string{"from": "by-object"}) + return cmp.Diff(expected, o.ByObject[pod].Label) + }, + }, + { + name: "ByObject.Fields gets defaulted from DefaultFieldSelector", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{"from": "default-field-selector"}), + }, + + verification: func(o Options) string { + expected := fields.SelectorFromSet(map[string]string{"from": "default-field-selector"}) + return cmp.Diff(expected, o.ByObject[pod].Field, cmp.Exporter(func(reflect.Type) bool { return true })) + }, + }, + { + name: "ByObject.Fields doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {Field: fields.SelectorFromSet(map[string]string{"from": "by-object"})}}, + DefaultFieldSelector: fields.SelectorFromSet(map[string]string{"from": "default-field-selector"}), + }, + + verification: func(o Options) string { + expected := fields.SelectorFromSet(map[string]string{"from": "by-object"}) + return cmp.Diff(expected, o.ByObject[pod].Field, cmp.Exporter(func(reflect.Type) bool { return true })) + }, + }, + { + name: "ByObject.UnsafeDisableDeepCopy gets defaulted from DefaultUnsafeDisableDeepCopy", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultUnsafeDisableDeepCopy: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(true) + return cmp.Diff(expected, o.ByObject[pod].UnsafeDisableDeepCopy) + }, + }, + { + name: "ByObject.UnsafeDisableDeepCopy doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {UnsafeDisableDeepCopy: ptr.To(false)}}, + DefaultUnsafeDisableDeepCopy: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(false) + return cmp.Diff(expected, o.ByObject[pod].UnsafeDisableDeepCopy) + }, + }, + { + name: "ByObject.EnableWatchBookmarks gets defaulted from DefaultEnableWatchBookmarks", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {}}, + DefaultEnableWatchBookmarks: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(true) + return cmp.Diff(expected, o.ByObject[pod].EnableWatchBookmarks) + }, + }, + { + name: "ByObject.EnableWatchBookmarks doesn't get defaulted when set", + in: Options{ + ByObject: map[client.Object]ByObject{pod: {EnableWatchBookmarks: ptr.To(false)}}, + DefaultEnableWatchBookmarks: ptr.To(true), + }, + + verification: func(o Options) string { + expected := ptr.To(false) + return cmp.Diff(expected, o.ByObject[pod].EnableWatchBookmarks) + }, + }, + { + name: "DefaultNamespace label selector gets defaulted from DefaultLabelSelector", + in: Options{ + DefaultNamespaces: map[string]Config{"default": {}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"})}, + } + return cmp.Diff(expected, o.DefaultNamespaces) + }, + }, + { + name: "ByObject.Namespaces get selector from DefaultNamespaces before DefaultSelector", + in: Options{ + ByObject: map[client.Object]ByObject{ + pod: {Namespaces: map[string]Config{"default": {}}}, + }, + DefaultNamespaces: map[string]Config{"default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "namespace"})}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default"}), + }, + + verification: func(o Options) string { + expected := Options{ + ByObject: map[client.Object]ByObject{ + pod: {Namespaces: map[string]Config{"default": { + LabelSelector: labels.SelectorFromSet(map[string]string{"from": "namespace"}), + }}}, + }, + DefaultNamespaces: map[string]Config{"default": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "namespace"})}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default"}), + } + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in DefaultNamespaces with custom selection logic", + in: Options{DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {}, + }}, + + verification: func(o Options) string { + expected := Options{ + DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie("metadata.namespace!=kube-public,metadata.namespace!=kube-system")}, + }, + } + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in DefaultNamespaces with custom selection logic and namespace default has its own field selector", + in: Options{DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie("spec.nodeName=foo")}, + }}, + + verification: func(o Options) string { + expected := Options{ + DefaultNamespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie( + "metadata.namespace!=kube-public,metadata.namespace!=kube-system,spec.nodeName=foo", + )}, + }, + } + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in ByObject.Namespaces with custom selection logic", + in: Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {}, + }, + }}}, + + verification: func(o Options) string { + expected := Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie( + "metadata.namespace!=kube-public,metadata.namespace!=kube-system", + )}, + }, + }}} + + return compare(expected, o) + }, + }, + { + name: "Two namespaces in ByObject.Namespaces with custom selection logic and namespace default has its own field selector", + in: Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie("spec.nodeName=foo")}, + }, + }}}, + + verification: func(o Options) string { + expected := Options{ByObject: map[client.Object]ByObject{pod: { + Namespaces: map[string]Config{ + "kube-public": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-public"})}, + "kube-system": {LabelSelector: labels.SelectorFromSet(map[string]string{"from": "kube-system"})}, + "": {FieldSelector: fields.ParseSelectorOrDie( + "metadata.namespace!=kube-public,metadata.namespace!=kube-system,spec.nodeName=foo", + )}, + }, + }}} + + return compare(expected, o) + }, + }, + { + name: "DefaultNamespace label selector doesn't get defaulted when set", + in: Options{ + DefaultNamespaces: map[string]Config{"default": {LabelSelector: labels.Everything()}}, + DefaultLabelSelector: labels.SelectorFromSet(map[string]string{"from": "default-label-selector"}), + }, + + verification: func(o Options) string { + expected := map[string]Config{ + "default": {LabelSelector: labels.Everything()}, + } + return cmp.Diff(expected, o.DefaultNamespaces) + }, + }, + { + name: "Defaulted namespaces in ByObject contain ByObject's selector", + in: Options{ + ByObject: map[client.Object]ByObject{ + pod: {Label: labels.SelectorFromSet(map[string]string{"from": "pod"})}, + }, + DefaultNamespaces: map[string]Config{"default": {}}, + }, + verification: func(o Options) string { + expected := Options{ + ByObject: map[client.Object]ByObject{ + pod: { + Label: labels.SelectorFromSet(map[string]string{"from": "pod"}), + Namespaces: map[string]Config{"default": { + LabelSelector: labels.SelectorFromSet(map[string]string{"from": "pod"}), + }}, + }, + }, + + DefaultNamespaces: map[string]Config{"default": {}}, + } + return compare(expected, o) + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.in.Mapper = &fakeRESTMapper{} + + defaulted, err := defaultOpts(&rest.Config{}, tc.in) + if err != nil { + t.Fatal(err) + } + + if diff := tc.verification(defaulted); diff != "" { + t.Errorf("expected config differs from actual: %s", diff) + } + }) + } +} + +func TestDefaultOptsRace(t *testing.T) { + opts := Options{ + Mapper: &fakeRESTMapper{}, + ByObject: map[client.Object]ByObject{ + &corev1.Pod{}: { + Label: labels.SelectorFromSet(map[string]string{"from": "pod"}), + Namespaces: map[string]Config{"default": { + LabelSelector: labels.SelectorFromSet(map[string]string{"from": "pod"}), + }}, + }, + }, + DefaultNamespaces: map[string]Config{"default": {}}, + } + + // Start go routines which re-use the above options struct. + wg := sync.WaitGroup{} + for range 2 { + wg.Add(1) + go func() { + _, _ = defaultOpts(&rest.Config{}, opts) + wg.Done() + }() + } + + // Wait for the go routines to finish. + wg.Wait() +} + +type fakeRESTMapper struct { + meta.RESTMapper +} + +func (f *fakeRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + return &meta.RESTMapping{Scope: meta.RESTScopeNamespace}, nil +} + +func TestDefaultConfigConsidersAllFields(t *testing.T) { + t.Parallel() + seed := time.Now().UnixNano() + t.Logf("Seed is %d", seed) + f := fuzz.NewWithSeed(seed).Funcs( + func(ls *labels.Selector, _ fuzz.Continue) { + *ls = labels.SelectorFromSet(map[string]string{"foo": "bar"}) + }, + func(fs *fields.Selector, _ fuzz.Continue) { + *fs = fields.SelectorFromSet(map[string]string{"foo": "bar"}) + }, + func(tf *cache.TransformFunc, _ fuzz.Continue) { + // never default this, as functions can not be compared so we fail down the line + }, + ) + + for i := 0; i < 100; i++ { + fuzzed := Config{} + f.Fuzz(&fuzzed) + + defaulted := defaultConfig(Config{}, fuzzed) + + if diff := cmp.Diff(fuzzed, defaulted, cmp.Exporter(func(reflect.Type) bool { return true })); diff != "" { + t.Errorf("Defaulted config doesn't match fuzzed one: %s", diff) + } + } +} diff --git a/pkg/cache/delegating_by_gvk_cache.go b/pkg/cache/delegating_by_gvk_cache.go new file mode 100644 index 0000000000..46bd243c66 --- /dev/null +++ b/pkg/cache/delegating_by_gvk_cache.go @@ -0,0 +1,136 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cache + +import ( + "context" + "maps" + "slices" + "strings" + "sync" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +// delegatingByGVKCache delegates to a type-specific cache if present +// and uses the defaultCache otherwise. +type delegatingByGVKCache struct { + scheme *runtime.Scheme + caches map[schema.GroupVersionKind]Cache + defaultCache Cache +} + +func (dbt *delegatingByGVKCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return err + } + return cache.Get(ctx, key, obj, opts...) +} + +func (dbt *delegatingByGVKCache) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + cache, err := dbt.cacheForObject(list) + if err != nil { + return err + } + return cache.List(ctx, list, opts...) +} + +func (dbt *delegatingByGVKCache) RemoveInformer(ctx context.Context, obj client.Object) error { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return err + } + return cache.RemoveInformer(ctx, obj) +} + +func (dbt *delegatingByGVKCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return nil, err + } + return cache.GetInformer(ctx, obj, opts...) +} + +func (dbt *delegatingByGVKCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) { + return dbt.cacheForGVK(gvk).GetInformerForKind(ctx, gvk, opts...) +} + +func (dbt *delegatingByGVKCache) Start(ctx context.Context) error { + allCaches := slices.Collect(maps.Values(dbt.caches)) + allCaches = append(allCaches, dbt.defaultCache) + + wg := &sync.WaitGroup{} + errs := make(chan error) + for idx := range allCaches { + cache := allCaches[idx] + wg.Add(1) + go func() { + defer wg.Done() + if err := cache.Start(ctx); err != nil { + errs <- err + } + }() + } + + select { + case err := <-errs: + return err + case <-ctx.Done(): + wg.Wait() + return nil + } +} + +func (dbt *delegatingByGVKCache) WaitForCacheSync(ctx context.Context) bool { + synced := true + for _, cache := range append(slices.Collect(maps.Values(dbt.caches)), dbt.defaultCache) { + if !cache.WaitForCacheSync(ctx) { + synced = false + } + } + + return synced +} + +func (dbt *delegatingByGVKCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + cache, err := dbt.cacheForObject(obj) + if err != nil { + return err + } + return cache.IndexField(ctx, obj, field, extractValue) +} + +func (dbt *delegatingByGVKCache) cacheForObject(o runtime.Object) (Cache, error) { + gvk, err := apiutil.GVKForObject(o, dbt.scheme) + if err != nil { + return nil, err + } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + return dbt.cacheForGVK(gvk), nil +} + +func (dbt *delegatingByGVKCache) cacheForGVK(gvk schema.GroupVersionKind) Cache { + if specific, hasSpecific := dbt.caches[gvk]; hasSpecific { + return specific + } + + return dbt.defaultCache +} diff --git a/pkg/cache/informer_cache.go b/pkg/cache/informer_cache.go index 771244d52a..091667b7fa 100644 --- a/pkg/cache/informer_cache.go +++ b/pkg/cache/informer_cache.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache/internal" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -45,11 +46,28 @@ func (*ErrCacheNotStarted) Error() string { return "the cache is not started, can not read objects" } +var _ error = (*ErrCacheNotStarted)(nil) + +// ErrResourceNotCached indicates that the resource type +// the client asked the cache for is not cached, i.e. the +// corresponding informer does not exist yet. +type ErrResourceNotCached struct { + GVK schema.GroupVersionKind +} + +// Error returns the error +func (r ErrResourceNotCached) Error() string { + return fmt.Sprintf("%s is not cached", r.GVK.String()) +} + +var _ error = (*ErrResourceNotCached)(nil) + // informerCache is a Kubernetes Object cache populated from internal.Informers. // informerCache wraps internal.Informers. type informerCache struct { scheme *runtime.Scheme *internal.Informers + readerFailOnMissingInformer bool } // Get implements Reader. @@ -59,7 +77,7 @@ func (ic *informerCache) Get(ctx context.Context, key client.ObjectKey, out clie return err } - started, cache, err := ic.Informers.Get(ctx, gvk, out) + started, cache, err := ic.getInformerForKind(ctx, gvk, out) if err != nil { return err } @@ -67,7 +85,7 @@ func (ic *informerCache) Get(ctx context.Context, key client.ObjectKey, out clie if !started { return &ErrCacheNotStarted{} } - return cache.Reader.Get(ctx, key, out) + return cache.Reader.Get(ctx, key, out, opts...) } // List implements Reader. @@ -77,7 +95,7 @@ func (ic *informerCache) List(ctx context.Context, out client.ObjectList, opts . return err } - started, cache, err := ic.Informers.Get(ctx, *gvk, cacheTypeObj) + started, cache, err := ic.getInformerForKind(ctx, *gvk, cacheTypeObj) if err != nil { return err } @@ -123,33 +141,64 @@ func (ic *informerCache) objectTypeForListObject(list client.ObjectList) (*schem return &gvk, cacheTypeObj, nil } -// GetInformerForKind returns the informer for the GroupVersionKind. -func (ic *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) { +func applyGetOptions(opts ...InformerGetOption) *internal.GetOptions { + cfg := &InformerGetOptions{} + for _, opt := range opts { + opt(cfg) + } + return (*internal.GetOptions)(cfg) +} + +// GetInformerForKind returns the informer for the GroupVersionKind. If no informer exists, one will be started. +func (ic *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) { // Map the gvk to an object obj, err := ic.scheme.New(gvk) if err != nil { return nil, err } - _, i, err := ic.Informers.Get(ctx, gvk, obj) + _, i, err := ic.Informers.Get(ctx, gvk, obj, applyGetOptions(opts...)) if err != nil { return nil, err } - return i.Informer, err + return i.Informer, nil } -// GetInformer returns the informer for the obj. -func (ic *informerCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) { +// GetInformer returns the informer for the obj. If no informer exists, one will be started. +func (ic *informerCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) { gvk, err := apiutil.GVKForObject(obj, ic.scheme) if err != nil { return nil, err } - _, i, err := ic.Informers.Get(ctx, gvk, obj) + _, i, err := ic.Informers.Get(ctx, gvk, obj, applyGetOptions(opts...)) if err != nil { return nil, err } - return i.Informer, err + return i.Informer, nil +} + +func (ic *informerCache) getInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *internal.Cache, error) { + if ic.readerFailOnMissingInformer { + cache, started, ok := ic.Informers.Peek(gvk, obj) + if !ok { + return false, nil, &ErrResourceNotCached{GVK: gvk} + } + return started, cache, nil + } + + return ic.Informers.Get(ctx, gvk, obj, &internal.GetOptions{}) +} + +// RemoveInformer deactivates and removes the informer from the cache. +func (ic *informerCache) RemoveInformer(_ context.Context, obj client.Object) error { + gvk, err := apiutil.GVKForObject(obj, ic.scheme) + if err != nil { + return err + } + + ic.Informers.Remove(gvk, obj) + return nil } // NeedLeaderElection implements the LeaderElectionRunnable interface @@ -158,11 +207,11 @@ func (ic *informerCache) NeedLeaderElection() bool { return false } -// IndexField adds an indexer to the underlying cache, using extraction function to get -// value(s) from the given field. This index can then be used by passing a field selector +// IndexField adds an indexer to the underlying informer, using extractValue function to get +// value(s) from the given field. This index can then be used by passing a field selector // to List. For one-to-one compatibility with "normal" field selectors, only return one value. -// The values may be anything. They will automatically be prefixed with the namespace of the -// given object, if present. The objects passed are guaranteed to be objects of the correct type. +// The values may be anything. They will automatically be prefixed with the namespace of the +// given object, if present. The objects passed are guaranteed to be objects of the correct type. func (ic *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { informer, err := ic.GetInformer(ctx, obj) if err != nil { @@ -171,7 +220,7 @@ func (ic *informerCache) IndexField(ctx context.Context, obj client.Object, fiel return indexByField(informer, field, extractValue) } -func indexByField(indexer Informer, field string, extractor client.IndexerFunc) error { +func indexByField(informer Informer, field string, extractValue client.IndexerFunc) error { indexFunc := func(objRaw interface{}) ([]string, error) { // TODO(directxman12): check if this is the correct type? obj, isObj := objRaw.(client.Object) @@ -184,7 +233,7 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc) } ns := meta.GetNamespace() - rawVals := extractor(obj) + rawVals := extractValue(obj) var vals []string if ns == "" { // if we're not doubling the keys for the namespaced case, just create a new slice with same length @@ -207,5 +256,5 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc) return vals, nil } - return indexer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc}) + return informer.AddIndexers(cache.Indexers{internal.FieldIndexName(field): indexFunc}) } diff --git a/pkg/cache/informertest/fake_cache.go b/pkg/cache/informertest/fake_cache.go index da3bf8e0d4..a1a442316f 100644 --- a/pkg/cache/informertest/fake_cache.go +++ b/pkg/cache/informertest/fake_cache.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" @@ -39,7 +40,7 @@ type FakeInformers struct { } // GetInformerForKind implements Informers. -func (c *FakeInformers) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (cache.Informer, error) { +func (c *FakeInformers) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...cache.InformerGetOption) (cache.Informer, error) { if c.Scheme == nil { c.Scheme = scheme.Scheme } @@ -52,14 +53,7 @@ func (c *FakeInformers) GetInformerForKind(ctx context.Context, gvk schema.Group // FakeInformerForKind implements Informers. func (c *FakeInformers) FakeInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (*controllertest.FakeInformer, error) { - if c.Scheme == nil { - c.Scheme = scheme.Scheme - } - obj, err := c.Scheme.New(gvk) - if err != nil { - return nil, err - } - i, err := c.informerFor(gvk, obj) + i, err := c.GetInformerForKind(ctx, gvk) if err != nil { return nil, err } @@ -67,7 +61,7 @@ func (c *FakeInformers) FakeInformerForKind(ctx context.Context, gvk schema.Grou } // GetInformer implements Informers. -func (c *FakeInformers) GetInformer(ctx context.Context, obj client.Object) (cache.Informer, error) { +func (c *FakeInformers) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { if c.Scheme == nil { c.Scheme = scheme.Scheme } @@ -79,6 +73,20 @@ func (c *FakeInformers) GetInformer(ctx context.Context, obj client.Object) (cac return c.informerFor(gvk, obj) } +// RemoveInformer implements Informers. +func (c *FakeInformers) RemoveInformer(ctx context.Context, obj client.Object) error { + if c.Scheme == nil { + c.Scheme = scheme.Scheme + } + gvks, _, err := c.Scheme.ObjectKinds(obj) + if err != nil { + return err + } + gvk := gvks[0] + delete(c.InformersByGVK, gvk) + return nil +} + // WaitForCacheSync implements Informers. func (c *FakeInformers) WaitForCacheSync(ctx context.Context) bool { if c.Synced == nil { @@ -88,16 +96,8 @@ func (c *FakeInformers) WaitForCacheSync(ctx context.Context) bool { } // FakeInformerFor implements Informers. -func (c *FakeInformers) FakeInformerFor(obj runtime.Object) (*controllertest.FakeInformer, error) { - if c.Scheme == nil { - c.Scheme = scheme.Scheme - } - gvks, _, err := c.Scheme.ObjectKinds(obj) - if err != nil { - return nil, err - } - gvk := gvks[0] - i, err := c.informerFor(gvk, obj) +func (c *FakeInformers) FakeInformerFor(ctx context.Context, obj client.Object) (*controllertest.FakeInformer, error) { + i, err := c.GetInformer(ctx, obj) if err != nil { return nil, err } diff --git a/pkg/cache/internal/cache_reader.go b/pkg/cache/internal/cache_reader.go index 3c8355bbde..eb6b544855 100644 --- a/pkg/cache/internal/cache_reader.go +++ b/pkg/cache/internal/cache_reader.go @@ -23,6 +23,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -35,7 +36,7 @@ import ( // CacheReader is a client.Reader. var _ client.Reader = &CacheReader{} -// CacheReader wraps a cache.Index to implement the client.CacheReader interface for a single type. +// CacheReader wraps a cache.Index to implement the client.Reader interface for a single type. type CacheReader struct { // indexer is the underlying indexer wrapped by this cache. indexer cache.Indexer @@ -54,6 +55,9 @@ type CacheReader struct { // Get checks the indexer for the object and writes a copy of it if found. func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error { + getOpts := client.GetOptions{} + getOpts.ApplyOptions(opts) + if c.scopeName == apimeta.RESTScopeNameRoot { key.Namespace = "" } @@ -67,9 +71,9 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob // Not found, return an error if !exists { - // Resource gets transformed into Kind in the error anyway, so this is fine return apierrors.NewNotFound(schema.GroupResource{ - Group: c.groupVersionKind.Group, + Group: c.groupVersionKind.Group, + // Resource gets set as Kind in the error so this is fine Resource: c.groupVersionKind.Kind, }, key.Name) } @@ -80,7 +84,7 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob return fmt.Errorf("cache contained %T, which is not an Object", obj) } - if c.disableDeepCopy { + if c.disableDeepCopy || (getOpts.UnsafeDisableDeepCopy != nil && *getOpts.UnsafeDisableDeepCopy) { // skip deep copy which might be unsafe // you must DeepCopy any object before mutating it outside } else { @@ -96,7 +100,7 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob return fmt.Errorf("cache had type %s, but %s was asked for", objVal.Type(), outVal.Type()) } reflect.Indirect(outVal).Set(reflect.Indirect(objVal)) - if !c.disableDeepCopy { + if !c.disableDeepCopy && (getOpts.UnsafeDisableDeepCopy == nil || !*getOpts.UnsafeDisableDeepCopy) { out.GetObjectKind().SetGroupVersionKind(c.groupVersionKind) } @@ -111,18 +115,20 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) + if listOpts.Continue != "" { + return fmt.Errorf("continue list option is not supported by the cache") + } + switch { case listOpts.FieldSelector != nil: - // TODO(directxman12): support more complicated field selectors by - // combining multiple indices, GetIndexers, etc - field, val, requiresExact := selector.RequiresExactMatch(listOpts.FieldSelector) + requiresExact := selector.RequiresExactMatch(listOpts.FieldSelector) if !requiresExact { return fmt.Errorf("non-exact field matches are not supported by the cache") } - // list all objects by the field selector. If this is namespaced and we have one, ask for the - // namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces" + // list all objects by the field selector. If this is namespaced and we have one, ask for the + // namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces" // namespace. - objs, err = c.indexer.ByIndex(FieldIndexName(field), KeyToNamespacedKey(listOpts.Namespace, val)) + objs, err = byIndexes(c.indexer, listOpts.FieldSelector.Requirements(), listOpts.Namespace) case listOpts.Namespace != "": objs, err = c.indexer.ByIndex(cache.NamespaceIndex, listOpts.Namespace) default: @@ -171,11 +177,65 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli } runtimeObjs = append(runtimeObjs, outObj) } - return apimeta.SetList(out, runtimeObjs) + + if err := apimeta.SetList(out, runtimeObjs); err != nil { + return err + } + + out.SetContinue("continue-not-supported") + return nil +} + +func byIndexes(indexer cache.Indexer, requires fields.Requirements, namespace string) ([]interface{}, error) { + var ( + err error + objs []interface{} + vals []string + ) + indexers := indexer.GetIndexers() + for idx, req := range requires { + indexName := FieldIndexName(req.Field) + indexedValue := KeyToNamespacedKey(namespace, req.Value) + if idx == 0 { + // we use first require to get snapshot data + // TODO(halfcrazy): use complicated index when client-go provides byIndexes + // https://github.com/kubernetes/kubernetes/issues/109329 + objs, err = indexer.ByIndex(indexName, indexedValue) + if err != nil { + return nil, err + } + if len(objs) == 0 { + return nil, nil + } + continue + } + fn, exist := indexers[indexName] + if !exist { + return nil, fmt.Errorf("index with name %s does not exist", indexName) + } + filteredObjects := make([]interface{}, 0, len(objs)) + for _, obj := range objs { + vals, err = fn(obj) + if err != nil { + return nil, err + } + for _, val := range vals { + if val == indexedValue { + filteredObjects = append(filteredObjects, obj) + break + } + } + } + if len(filteredObjects) == 0 { + return nil, nil + } + objs = filteredObjects + } + return objs, nil } // objectKeyToStorageKey converts an object key to store key. -// It's akin to MetaNamespaceKeyFunc. It's separate from +// It's akin to MetaNamespaceKeyFunc. It's separate from // String to allow keeping the key format easily in sync with // MetaNamespaceKeyFunc. func objectKeyToStoreKey(k client.ObjectKey) string { @@ -191,7 +251,7 @@ func FieldIndexName(field string) string { return "field:" + field } -// noNamespaceNamespace is used as the "namespace" when we want to list across all namespaces. +// allNamespacesNamespace is used as the "namespace" when we want to list across all namespaces. const allNamespacesNamespace = "__all_namespaces" // KeyToNamespacedKey prefixes the given index key with a namespace diff --git a/pkg/cache/internal/informers.go b/pkg/cache/internal/informers.go index 09e0111114..f216be0d9e 100644 --- a/pkg/cache/internal/informers.go +++ b/pkg/cache/internal/informers.go @@ -18,46 +18,54 @@ package internal import ( "context" + "errors" "fmt" "math/rand" "net/http" "sync" "time" + "github.com/go-logr/logr" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" "k8s.io/client-go/metadata" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" + "sigs.k8s.io/controller-runtime/pkg/internal/syncs" ) +var log = logf.RuntimeLog.WithName("cache") + // InformersOpts configures an InformerMap. type InformersOpts struct { - HTTPClient *http.Client - Scheme *runtime.Scheme - Mapper meta.RESTMapper - ResyncPeriod time.Duration - Namespace string - ByGVK map[schema.GroupVersionKind]InformersOptsByGVK -} - -// InformersOptsByGVK configured additional by group version kind (or object) -// in an InformerMap. -type InformersOptsByGVK struct { + HTTPClient *http.Client + Scheme *runtime.Scheme + Mapper meta.RESTMapper + ResyncPeriod time.Duration + Namespace string + NewInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer Selector Selector Transform cache.TransformFunc - UnsafeDisableDeepCopy *bool + UnsafeDisableDeepCopy bool + EnableWatchBookmarks bool + WatchErrorHandler cache.WatchErrorHandlerWithContext } // NewInformers creates a new InformersMap that can create informers under the hood. func NewInformers(config *rest.Config, options *InformersOpts) *Informers { + newInformer := cache.NewSharedIndexInformer + if options.NewInformer != nil { + newInformer = options.NewInformer + } return &Informers{ config: config, httpClient: options.HTTPClient, @@ -68,12 +76,17 @@ func NewInformers(config *rest.Config, options *InformersOpts) *Informers { Unstructured: make(map[schema.GroupVersionKind]*Cache), Metadata: make(map[schema.GroupVersionKind]*Cache), }, - codecs: serializer.NewCodecFactory(options.Scheme), - paramCodec: runtime.NewParameterCodec(options.Scheme), - resync: options.ResyncPeriod, - startWait: make(chan struct{}), - namespace: options.Namespace, - byGVK: options.ByGVK, + codecs: serializer.NewCodecFactory(options.Scheme), + paramCodec: runtime.NewParameterCodec(options.Scheme), + resync: options.ResyncPeriod, + startWait: make(chan struct{}), + namespace: options.Namespace, + selector: options.Selector, + transform: options.Transform, + unsafeDisableDeepCopy: options.UnsafeDisableDeepCopy, + enableWatchBookmarks: options.EnableWatchBookmarks, + newInformer: newInformer, + watchErrorHandler: options.WatchErrorHandler, } } @@ -84,6 +97,21 @@ type Cache struct { // CacheReader wraps Informer and implements the CacheReader interface for a single type Reader CacheReader + + // Stop can be used to stop this individual informer. + stop chan struct{} +} + +// Start starts the informer managed by a MapEntry. +// Blocks until the informer stops. The informer can be stopped +// either individually (via the entry's stop channel) or globally +// via the provided stop argument. +func (c *Cache) Start(stop <-chan struct{}) { + // Stop on either the whole map stopping or just this informer being removed. + internalStop, cancel := syncs.MergeChans(stop, c.stop) + defer cancel() + // Convert the stop channel to a context and then add the logger. + c.Informer.RunWithContext(logr.NewContext(wait.ContextForChannel(internalStop), log)) } type tracker struct { @@ -92,6 +120,13 @@ type tracker struct { Metadata map[schema.GroupVersionKind]*Cache } +// GetOptions provides configuration to customize the behavior when +// getting an informer. +type GetOptions struct { + // BlockUntilSynced controls if the informer retrieval will block until the informer is synced. Defaults to `true`. + BlockUntilSynced *bool +} + // Informers create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. // It uses a standard parameter codec constructed based on the given generated Scheme. type Informers struct { @@ -144,73 +179,53 @@ type Informers struct { // default or empty string means all namespaces namespace string - byGVK map[schema.GroupVersionKind]InformersOptsByGVK -} + selector Selector + transform cache.TransformFunc + unsafeDisableDeepCopy bool + enableWatchBookmarks bool -func (ip *Informers) getSelector(gvk schema.GroupVersionKind) Selector { - if ip.byGVK == nil { - return Selector{} - } - if res, ok := ip.byGVK[gvk]; ok { - return res.Selector - } - if res, ok := ip.byGVK[schema.GroupVersionKind{}]; ok { - return res.Selector - } - return Selector{} -} - -func (ip *Informers) getTransform(gvk schema.GroupVersionKind) cache.TransformFunc { - if ip.byGVK == nil { - return nil - } - if res, ok := ip.byGVK[gvk]; ok { - return res.Transform - } - if res, ok := ip.byGVK[schema.GroupVersionKind{}]; ok { - return res.Transform - } - return nil -} + // NewInformer allows overriding of the shared index informer constructor for testing. + newInformer func(cache.ListerWatcher, runtime.Object, time.Duration, cache.Indexers) cache.SharedIndexInformer -func (ip *Informers) getDisableDeepCopy(gvk schema.GroupVersionKind) bool { - if ip.byGVK == nil { - return false - } - if res, ok := ip.byGVK[gvk]; ok && res.UnsafeDisableDeepCopy != nil { - return *res.UnsafeDisableDeepCopy - } - if res, ok := ip.byGVK[schema.GroupVersionKind{}]; ok && res.UnsafeDisableDeepCopy != nil { - return *res.UnsafeDisableDeepCopy - } - return false + // watchErrorHandler allows the shared index informer's + // watchErrorHandler to be set by overriding the options + // or to use the default watchErrorHandler + watchErrorHandler cache.WatchErrorHandlerWithContext } -// Start calls Run on each of the informers and sets started to true. Blocks on the context. +// Start calls Run on each of the informers and sets started to true. Blocks on the context. // It doesn't return start because it can't return an error, and it's not a runnable directly. func (ip *Informers) Start(ctx context.Context) error { - func() { + if err := func() error { ip.mu.Lock() defer ip.mu.Unlock() + if ip.started { + return errors.New("informer already started") //nolint:stylecheck + } + // Set the context so it can be passed to informers that are added later ip.ctx = ctx // Start each informer for _, i := range ip.tracker.Structured { - ip.startInformerLocked(i.Informer) + ip.startInformerLocked(i) } for _, i := range ip.tracker.Unstructured { - ip.startInformerLocked(i.Informer) + ip.startInformerLocked(i) } for _, i := range ip.tracker.Metadata { - ip.startInformerLocked(i.Informer) + ip.startInformerLocked(i) } // Set started to true so we immediately start any informers added later. ip.started = true close(ip.startWait) - }() + + return nil + }(); err != nil { + return err + } <-ctx.Done() // Block until the context is done ip.mu.Lock() ip.stopped = true // Set stopped to true so we don't start any new informers @@ -219,7 +234,7 @@ func (ip *Informers) Start(ctx context.Context) error { return nil } -func (ip *Informers) startInformerLocked(informer cache.SharedIndexInformer) { +func (ip *Informers) startInformerLocked(cacheEntry *Cache) { // Don't start the informer in case we are already waiting for the items in // the waitGroup to finish, since waitGroups don't support waiting and adding // at the same time. @@ -230,7 +245,7 @@ func (ip *Informers) startInformerLocked(informer cache.SharedIndexInformer) { ip.waitGroup.Add(1) go func() { defer ip.waitGroup.Done() - informer.Run(ip.ctx.Done()) + cacheEntry.Start(ip.ctx.Done()) }() } @@ -271,18 +286,19 @@ func (ip *Informers) WaitForCacheSync(ctx context.Context) bool { return cache.WaitForCacheSync(ctx.Done(), ip.getHasSyncedFuncs()...) } -func (ip *Informers) get(gvk schema.GroupVersionKind, obj runtime.Object) (res *Cache, started bool, ok bool) { +// Peek attempts to get the informer for the GVK, but does not start one if one does not exist. +func (ip *Informers) Peek(gvk schema.GroupVersionKind, obj runtime.Object) (res *Cache, started bool, ok bool) { ip.mu.RLock() defer ip.mu.RUnlock() i, ok := ip.informersByType(obj)[gvk] return i, ip.started, ok } -// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns +// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns // the Informer from the map. -func (ip *Informers) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *Cache, error) { +func (ip *Informers) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object, opts *GetOptions) (bool, *Cache, error) { // Return the informer if it is found - i, started, ok := ip.get(gvk, obj) + i, started, ok := ip.Peek(gvk, obj) if !ok { var err error if i, started, err = ip.addInformerToMap(gvk, obj); err != nil { @@ -290,7 +306,12 @@ func (ip *Informers) Get(ctx context.Context, gvk schema.GroupVersionKind, obj r } } - if started && !i.Informer.HasSynced() { + shouldBlock := true + if opts.BlockUntilSynced != nil { + shouldBlock = *opts.BlockUntilSynced + } + + if shouldBlock && started && !i.Informer.HasSynced() { // Wait for it to sync before returning the Informer so that folks don't read from a stale cache. if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) { return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0) @@ -300,6 +321,21 @@ func (ip *Informers) Get(ctx context.Context, gvk schema.GroupVersionKind, obj r return started, i, nil } +// Remove removes an informer entry and stops it if it was running. +func (ip *Informers) Remove(gvk schema.GroupVersionKind, obj runtime.Object) { + ip.mu.Lock() + defer ip.mu.Unlock() + + informerMap := ip.informersByType(obj) + + entry, ok := informerMap[gvk] + if !ok { + return + } + close(entry.stop) + delete(informerMap, gvk) +} + func (ip *Informers) informersByType(obj runtime.Object) map[schema.GroupVersionKind]*Cache { switch obj.(type) { case runtime.Unstructured: @@ -311,11 +347,12 @@ func (ip *Informers) informersByType(obj runtime.Object) map[schema.GroupVersion } } +// addInformerToMap either returns an existing informer or creates a new informer, adds it to the map and returns it. func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.Object) (*Cache, bool, error) { ip.mu.Lock() defer ip.mu.Unlock() - // Check the cache to see if we already have an Informer. If we do, return the Informer. + // Check the cache to see if we already have an Informer. If we do, return the Informer. // This is for the case where 2 routines tried to get the informer when it wasn't in the map // so neither returned early, but the first one created it. if i, ok := ip.informersByType(obj)[gvk]; ok { @@ -327,22 +364,31 @@ func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.O if err != nil { return nil, false, err } - sharedIndexInformer := cache.NewSharedIndexInformer(&cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - ip.getSelector(gvk).ApplyToList(&opts) - return listWatcher.ListFunc(opts) + sharedIndexInformer := ip.newInformer(&cache.ListWatch{ + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { + ip.selector.ApplyToList(&opts) + return listWatcher.ListWithContextFunc(ctx, opts) }, - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.getSelector(gvk).ApplyToList(&opts) + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { opts.Watch = true // Watch needs to be set to true separately - return listWatcher.WatchFunc(opts) + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + + ip.selector.ApplyToList(&opts) + return listWatcher.WatchFuncWithContext(ctx, opts) }, }, obj, calculateResyncPeriod(ip.resync), cache.Indexers{ cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, }) + // Set WatchErrorHandler on SharedIndexInformer if set + if ip.watchErrorHandler != nil { + if err := sharedIndexInformer.SetWatchErrorHandlerWithContext(ip.watchErrorHandler); err != nil { + return nil, false, err + } + } + // Check to see if there is a transformer for this gvk - if err := sharedIndexInformer.SetTransform(ip.getTransform(gvk)); err != nil { + if err := sharedIndexInformer.SetTransform(ip.transform); err != nil { return nil, false, err } @@ -358,15 +404,16 @@ func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.O indexer: sharedIndexInformer.GetIndexer(), groupVersionKind: gvk, scopeName: mapping.Scope.Name(), - disableDeepCopy: ip.getDisableDeepCopy(gvk), + disableDeepCopy: ip.unsafeDisableDeepCopy, }, + stop: make(chan struct{}), } ip.informersByType(obj)[gvk] = i // Start the informer in case the InformersMap has started, otherwise it will be // started when the InformersMap starts. if ip.started { - ip.startInformerLocked(i.Informer) + ip.startInformerLocked(i) } return i, ip.started, nil } @@ -382,7 +429,7 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob // Figure out if the GVK we're dealing with is global, or namespace scoped. var namespace string if mapping.Scope.Name() == meta.RESTScopeNameNamespace { - namespace = restrictNamespaceBySelector(ip.namespace, ip.getSelector(gvk)) + namespace = restrictNamespaceBySelector(ip.namespace, ip.selector) } switch obj.(type) { @@ -400,18 +447,21 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob } resources := dynamicClient.Resource(mapping.Resource) return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { if namespace != "" { - return resources.Namespace(namespace).List(ip.ctx, opts) + return resources.Namespace(namespace).List(ctx, opts) } - return resources.List(ip.ctx, opts) + return resources.List(ctx, opts) }, // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + if namespace != "" { - return resources.Namespace(namespace).Watch(ip.ctx, opts) + return resources.Namespace(namespace).Watch(ctx, opts) } - return resources.Watch(ip.ctx, opts) + return resources.Watch(ctx, opts) }, }, nil // @@ -431,15 +481,15 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob resources := metadataClient.Resource(mapping.Resource) return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { var ( list *metav1.PartialObjectMetadataList err error ) if namespace != "" { - list, err = resources.Namespace(namespace).List(ip.ctx, opts) + list, err = resources.Namespace(namespace).List(ctx, opts) } else { - list, err = resources.List(ip.ctx, opts) + list, err = resources.List(ctx, opts) } if list != nil { for i := range list.Items { @@ -449,11 +499,14 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob return list, err }, // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watcher watch.Interface, err error) { + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watcher watch.Interface, err error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + if namespace != "" { - watcher, err = resources.Namespace(namespace).Watch(ip.ctx, opts) + watcher, err = resources.Namespace(namespace).Watch(ctx, opts) } else { - watcher, err = resources.Watch(ip.ctx, opts) + watcher, err = resources.Watch(ctx, opts) } if err != nil { return nil, err @@ -465,7 +518,7 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob // Structured. // default: - client, err := apiutil.RESTClientForGVK(gvk, false, ip.config, ip.codecs, ip.httpClient) + client, err := apiutil.RESTClientForGVK(gvk, false, false, ip.config, ip.codecs, ip.httpClient) if err != nil { return nil, err } @@ -475,7 +528,7 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob return nil, err } return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + ListWithContextFunc: func(ctx context.Context, opts metav1.ListOptions) (runtime.Object, error) { // Build the request. req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec) if namespace != "" { @@ -484,20 +537,23 @@ func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Ob // Create the resulting object, and execute the request. res := listObj.DeepCopyObject() - if err := req.Do(ip.ctx).Into(res); err != nil { + if err := req.Do(ctx).Into(res); err != nil { return nil, err } return res, nil }, // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + WatchFuncWithContext: func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + opts.Watch = true // Watch needs to be set to true separately + opts.AllowWatchBookmarks = ip.enableWatchBookmarks + // Build the request. req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec) if namespace != "" { req.Namespace(namespace) } // Call the watch. - return req.Watch(ip.ctx) + return req.Watch(ctx) }, }, nil } @@ -535,7 +591,7 @@ func newGVKFixupWatcher(gvk schema.GroupVersionKind, watcher watch.Interface) wa // hammer the apiserver with list requests simultaneously. func calculateResyncPeriod(resync time.Duration) time.Duration { // the factor will fall into [0.9, 1.1) - factor := rand.Float64()/5.0 + 0.9 //nolint:gosec + factor := rand.Float64()/5.0 + 0.9 return time.Duration(float64(resync.Nanoseconds()) * factor) } diff --git a/pkg/cache/internal/transformers.go b/pkg/cache/internal/transformers.go deleted file mode 100644 index 0725f550c5..0000000000 --- a/pkg/cache/internal/transformers.go +++ /dev/null @@ -1,55 +0,0 @@ -package internal - -import ( - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/tools/cache" - - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" -) - -// TransformFuncByGVK provides access to the correct transform function for -// any given GVK. -type TransformFuncByGVK interface { - Set(runtime.Object, *runtime.Scheme, cache.TransformFunc) error - Get(schema.GroupVersionKind) cache.TransformFunc - SetDefault(transformer cache.TransformFunc) -} - -type transformFuncByGVK struct { - defaultTransform cache.TransformFunc - transformers map[schema.GroupVersionKind]cache.TransformFunc -} - -// TransformFuncByGVKFromMap creates a TransformFuncByGVK from a map that -// maps GVKs to TransformFuncs. -func TransformFuncByGVKFromMap(in map[schema.GroupVersionKind]cache.TransformFunc) TransformFuncByGVK { - byGVK := &transformFuncByGVK{} - if defaultFunc, hasDefault := in[schema.GroupVersionKind{}]; hasDefault { - byGVK.defaultTransform = defaultFunc - } - delete(in, schema.GroupVersionKind{}) - byGVK.transformers = in - return byGVK -} - -func (t *transformFuncByGVK) SetDefault(transformer cache.TransformFunc) { - t.defaultTransform = transformer -} - -func (t *transformFuncByGVK) Set(obj runtime.Object, scheme *runtime.Scheme, transformer cache.TransformFunc) error { - gvk, err := apiutil.GVKForObject(obj, scheme) - if err != nil { - return err - } - - t.transformers[gvk] = transformer - return nil -} - -func (t transformFuncByGVK) Get(gvk schema.GroupVersionKind) cache.TransformFunc { - if val, ok := t.transformers[gvk]; ok { - return val - } - return t.defaultTransform -} diff --git a/pkg/cache/multi_namespace_cache.go b/pkg/cache/multi_namespace_cache.go index ac97beae94..d7d7b0e7c2 100644 --- a/pkg/cache/multi_namespace_cache.go +++ b/pkg/cache/multi_namespace_cache.go @@ -23,10 +23,11 @@ import ( corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) @@ -34,49 +35,31 @@ import ( // a new global namespaced cache to handle cluster scoped resources. const globalCache = "_cluster-scope" -// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache. -// This will scope the cache to a list of namespaces. Listing for all namespaces -// will list for all the namespaces that this knows about. By default this will create -// a global cache for cluster scoped resource. Note that this is not intended -// to be used for excluding namespaces, this is better done via a Predicate. Also note that -// you may face performance issues when using this with a high number of namespaces. -// -// Deprecated: Use cache.Options.Namespaces instead. -func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc { - return func(config *rest.Config, opts Options) (Cache, error) { - opts.Namespaces = namespaces - return newMultiNamespaceCache(config, opts) - } -} - -func newMultiNamespaceCache(config *rest.Config, opts Options) (Cache, error) { - if len(opts.Namespaces) < 2 { - return nil, fmt.Errorf("must specify more than one namespace to use multi-namespace cache") - } - opts, err := defaultOpts(config, opts) - if err != nil { - return nil, err - } - +func newMultiNamespaceCache( + newCache newCacheFunc, + scheme *runtime.Scheme, + restMapper apimeta.RESTMapper, + namespaces map[string]Config, + globalConfig *Config, // may be nil in which case no cache for cluster-scoped objects will be created +) Cache { // Create every namespace cache. caches := map[string]Cache{} - for _, ns := range opts.Namespaces { - opts.Namespaces = []string{ns} - c, err := New(config, opts) - if err != nil { - return nil, err - } - caches[ns] = c + for namespace, config := range namespaces { + caches[namespace] = newCache(config, namespace) } - // Create a cache for cluster scoped resources. - opts.Namespaces = []string{} - gCache, err := New(config, opts) - if err != nil { - return nil, fmt.Errorf("error creating global cache: %w", err) + // Create a cache for cluster scoped resources if requested + var clusterCache Cache + if globalConfig != nil { + clusterCache = newCache(*globalConfig, corev1.NamespaceAll) } - return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil + return &multiNamespaceCache{ + namespaceToCache: caches, + Scheme: scheme, + RESTMapper: restMapper, + clusterCache: clusterCache, + } } // multiNamespaceCache knows how to handle multiple namespaced caches @@ -84,108 +67,139 @@ func newMultiNamespaceCache(config *rest.Config, opts Options) (Cache, error) { // operator to a list of namespaces instead of watching every namespace // in the cluster. type multiNamespaceCache struct { - namespaceToCache map[string]Cache Scheme *runtime.Scheme RESTMapper apimeta.RESTMapper + namespaceToCache map[string]Cache clusterCache Cache } var _ Cache = &multiNamespaceCache{} // Methods for multiNamespaceCache to conform to the Informers interface. -func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) { - informers := map[string]Informer{} - // If the object is clusterscoped, get the informer from clusterCache, +func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object, opts ...InformerGetOption) (Informer, error) { + // If the object is cluster scoped, get the informer from clusterCache, // if not use the namespaced caches. isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) if err != nil { return nil, err } if !isNamespaced { - clusterCacheInf, err := c.clusterCache.GetInformer(ctx, obj) + clusterCacheInformer, err := c.clusterCache.GetInformer(ctx, obj, opts...) if err != nil { return nil, err } - informers[globalCache] = clusterCacheInf - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{ + namespaceToInformer: map[string]Informer{ + globalCache: clusterCacheInformer, + }, + }, nil } + namespaceToInformer := map[string]Informer{} for ns, cache := range c.namespaceToCache { - informer, err := cache.GetInformer(ctx, obj) + informer, err := cache.GetInformer(ctx, obj, opts...) if err != nil { return nil, err } - informers[ns] = informer + namespaceToInformer[ns] = informer } - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil } -func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) { - informers := map[string]Informer{} - +func (c *multiNamespaceCache) RemoveInformer(ctx context.Context, obj client.Object) error { // If the object is clusterscoped, get the informer from clusterCache, // if not use the namespaced caches. + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) + if err != nil { + return err + } + if !isNamespaced { + return c.clusterCache.RemoveInformer(ctx, obj) + } + + for _, cache := range c.namespaceToCache { + err := cache.RemoveInformer(ctx, obj) + if err != nil { + return err + } + } + + return nil +} + +func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...InformerGetOption) (Informer, error) { + // If the object is cluster scoped, get the informer from clusterCache, + // if not use the namespaced caches. isNamespaced, err := apiutil.IsGVKNamespaced(gvk, c.RESTMapper) if err != nil { return nil, err } if !isNamespaced { - clusterCacheInf, err := c.clusterCache.GetInformerForKind(ctx, gvk) + clusterCacheInformer, err := c.clusterCache.GetInformerForKind(ctx, gvk, opts...) if err != nil { return nil, err } - informers[globalCache] = clusterCacheInf - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{ + namespaceToInformer: map[string]Informer{ + globalCache: clusterCacheInformer, + }, + }, nil } + namespaceToInformer := map[string]Informer{} for ns, cache := range c.namespaceToCache { - informer, err := cache.GetInformerForKind(ctx, gvk) + informer, err := cache.GetInformerForKind(ctx, gvk, opts...) if err != nil { return nil, err } - informers[ns] = informer + namespaceToInformer[ns] = informer } - return &multiNamespaceInformer{namespaceToInformer: informers}, nil + return &multiNamespaceInformer{namespaceToInformer: namespaceToInformer}, nil } func (c *multiNamespaceCache) Start(ctx context.Context) error { + errs := make(chan error) // start global cache - go func() { - err := c.clusterCache.Start(ctx) - if err != nil { - log.Error(err, "cluster scoped cache failed to start") - } - }() + if c.clusterCache != nil { + go func() { + err := c.clusterCache.Start(ctx) + if err != nil { + errs <- fmt.Errorf("failed to start cluster-scoped cache: %w", err) + } + }() + } // start namespaced caches for ns, cache := range c.namespaceToCache { go func(ns string, cache Cache) { - err := cache.Start(ctx) - if err != nil { - log.Error(err, "multinamespace cache failed to start namespaced informer", "namespace", ns) + if err := cache.Start(ctx); err != nil { + errs <- fmt.Errorf("failed to start cache for namespace %s: %w", ns, err) } }(ns, cache) } - - <-ctx.Done() - return nil + select { + case <-ctx.Done(): + return nil + case err := <-errs: + return err + } } func (c *multiNamespaceCache) WaitForCacheSync(ctx context.Context) bool { synced := true for _, cache := range c.namespaceToCache { - if s := cache.WaitForCacheSync(ctx); !s { - synced = s + if !cache.WaitForCacheSync(ctx) { + synced = false } } // check if cluster scoped cache has synced - if !c.clusterCache.WaitForCacheSync(ctx) { + if c.clusterCache != nil && !c.clusterCache.WaitForCacheSync(ctx) { synced = false } return synced @@ -222,9 +236,12 @@ func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj cache, ok := c.namespaceToCache[key.Namespace] if !ok { + if global, hasGlobal := c.namespaceToCache[metav1.NamespaceAll]; hasGlobal { + return global.Get(ctx, key, obj, opts...) + } return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", key) } - return cache.Get(ctx, key, obj) + return cache.Get(ctx, key, obj, opts...) } // List multi namespace cache will get all the objects in the namespaces that the cache is watching if asked for all namespaces. @@ -232,6 +249,10 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) + if listOpts.Continue != "" { + return fmt.Errorf("continue list option is not supported by the cache") + } + isNamespaced, err := apiutil.IsObjectNamespaced(list, c.Scheme, c.RESTMapper) if err != nil { return err @@ -245,7 +266,10 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, if listOpts.Namespace != corev1.NamespaceAll { cache, ok := c.namespaceToCache[listOpts.Namespace] if !ok { - return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", listOpts.Namespace) + if global, hasGlobal := c.namespaceToCache[AllNamespaces]; hasGlobal { + return global.List(ctx, list, opts...) + } + return fmt.Errorf("unable to list: %v because of unknown namespace for the cache", listOpts.Namespace) } return cache.List(ctx, list, opts...) } @@ -255,10 +279,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, return err } - allItems, err := apimeta.ExtractList(list) - if err != nil { - return err - } + allItems := []runtime.Object{} limitSet := listOpts.Limit > 0 @@ -278,12 +299,14 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, return fmt.Errorf("object: %T must be a list type", list) } allItems = append(allItems, items...) + // The last list call should have the most correct resource version. resourceVersion = accessor.GetResourceVersion() if limitSet { // decrement Limit by the number of items // fetched from the current namespace. listOpts.Limit -= int64(len(items)) + // if a Limit was set and the number of // items read has reached this set limit, // then stop reading. @@ -294,7 +317,12 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, } listAccessor.SetResourceVersion(resourceVersion) - return apimeta.SetList(list, allItems) + if err := apimeta.SetList(list, allItems); err != nil { + return err + } + + list.SetContinue("continue-not-supported") + return nil } // multiNamespaceInformer knows how to handle interacting with the underlying informer across multiple namespaces. @@ -306,18 +334,11 @@ type handlerRegistration struct { handles map[string]toolscache.ResourceEventHandlerRegistration } -type syncer interface { - HasSynced() bool -} - // HasSynced asserts that the handler has been called for the full initial state of the informer. -// This uses syncer to be compatible between client-go 1.27+ and older versions when the interface changed. func (h handlerRegistration) HasSynced() bool { - for _, reg := range h.handles { - if s, ok := reg.(syncer); ok { - if !s.HasSynced() { - return false - } + for _, h := range h.handles { + if !h.HasSynced() { + return false } } return true @@ -325,9 +346,12 @@ func (h handlerRegistration) HasSynced() bool { var _ Informer = &multiNamespaceInformer{} -// AddEventHandler adds the handler to each namespaced informer. +// AddEventHandler adds the handler to each informer. func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) { - handles := handlerRegistration{handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer))} + handles := handlerRegistration{ + handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)), + } + for ns, informer := range i.namespaceToInformer { registration, err := informer.AddEventHandler(handler) if err != nil { @@ -335,12 +359,16 @@ func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEven } handles.handles[ns] = registration } + return handles, nil } // AddEventHandlerWithResyncPeriod adds the handler with a resync period to each namespaced informer. func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) { - handles := handlerRegistration{handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer))} + handles := handlerRegistration{ + handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)), + } + for ns, informer := range i.namespaceToInformer { registration, err := informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod) if err != nil { @@ -348,14 +376,32 @@ func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolsca } handles.handles[ns] = registration } + return handles, nil } -// RemoveEventHandler removes a formerly added event handler given by its registration handle. +// AddEventHandlerWithOptions adds the handler with options to each namespaced informer. +func (i *multiNamespaceInformer) AddEventHandlerWithOptions(handler toolscache.ResourceEventHandler, options toolscache.HandlerOptions) (toolscache.ResourceEventHandlerRegistration, error) { + handles := handlerRegistration{ + handles: make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)), + } + + for ns, informer := range i.namespaceToInformer { + registration, err := informer.AddEventHandlerWithOptions(handler, options) + if err != nil { + return nil, err + } + handles.handles[ns] = registration + } + + return handles, nil +} + +// RemoveEventHandler removes a previously added event handler given by its registration handle. func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHandlerRegistration) error { handles, ok := h.(handlerRegistration) if !ok { - return fmt.Errorf("it is not the registration returned by multiNamespaceInformer") + return fmt.Errorf("registration is not a registration returned by multiNamespaceInformer") } for ns, informer := range i.namespaceToInformer { registration, ok := handles.handles[ns] @@ -369,7 +415,7 @@ func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHa return nil } -// AddIndexers adds the indexer for each namespaced informer. +// AddIndexers adds the indexers to each informer. func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error { for _, informer := range i.namespaceToInformer { err := informer.AddIndexers(indexers) @@ -380,11 +426,21 @@ func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error return nil } -// HasSynced checks if each namespaced informer has synced. +// HasSynced checks if each informer has synced. func (i *multiNamespaceInformer) HasSynced() bool { for _, informer := range i.namespaceToInformer { - if ok := informer.HasSynced(); !ok { - return ok + if !informer.HasSynced() { + return false + } + } + return true +} + +// IsStopped checks if each namespaced informer has stopped, returns false if any are still running. +func (i *multiNamespaceInformer) IsStopped() bool { + for _, informer := range i.namespaceToInformer { + if stopped := informer.IsStopped(); !stopped { + return false } } return true diff --git a/pkg/certwatcher/certwatcher.go b/pkg/certwatcher/certwatcher.go index 2b9b60d8d7..2362d020b8 100644 --- a/pkg/certwatcher/certwatcher.go +++ b/pkg/certwatcher/certwatcher.go @@ -17,13 +17,16 @@ limitations under the License. package certwatcher import ( + "bytes" "context" "crypto/tls" "fmt" + "os" "sync" "time" "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -33,18 +36,25 @@ import ( var log = logf.RuntimeLog.WithName("certwatcher") -// CertWatcher watches certificate and key files for changes. When either file -// changes, it reads and parses both and calls an optional callback with the new -// certificate. +const defaultWatchInterval = 10 * time.Second + +// CertWatcher watches certificate and key files for changes. +// It always returns the cached version, +// but periodically reads and parses certificate and key for changes +// and calls an optional callback with the new certificate. type CertWatcher struct { sync.RWMutex currentCert *tls.Certificate watcher *fsnotify.Watcher + interval time.Duration + log logr.Logger certPath string keyPath string + cachedKeyPEMBlock []byte + // callback is a function to be invoked when the certificate changes. callback func(tls.Certificate) } @@ -56,6 +66,8 @@ func New(certPath, keyPath string) (*CertWatcher, error) { cw := &CertWatcher{ certPath: certPath, keyPath: keyPath, + interval: defaultWatchInterval, + log: log.WithValues("cert", certPath, "key", keyPath), } // Initial read of certificate and key. @@ -71,6 +83,12 @@ func New(certPath, keyPath string) (*CertWatcher, error) { return cw, nil } +// WithWatchInterval sets the watch interval and returns the CertWatcher pointer +func (cw *CertWatcher) WithWatchInterval(interval time.Duration) *CertWatcher { + cw.interval = interval + return cw +} + // RegisterCallback registers a callback to be invoked when the certificate changes. func (cw *CertWatcher) RegisterCallback(callback func(tls.Certificate)) { cw.Lock() @@ -112,12 +130,20 @@ func (cw *CertWatcher) Start(ctx context.Context) error { go cw.Watch() - log.Info("Starting certificate watcher") - - // Block until the context is done. - <-ctx.Done() + ticker := time.NewTicker(cw.interval) + defer ticker.Stop() - return cw.watcher.Close() + cw.log.Info("Starting certificate poll+watcher", "interval", cw.interval) + for { + select { + case <-ctx.Done(): + return cw.watcher.Close() + case <-ticker.C: + if err := cw.ReadCertificate(); err != nil { + cw.log.Error(err, "failed read certificate") + } + } + } } // Watch reads events from the watcher's channel and reacts to changes. @@ -131,34 +157,61 @@ func (cw *CertWatcher) Watch() { } cw.handleEvent(event) - case err, ok := <-cw.watcher.Errors: // Channel is closed. if !ok { return } - log.Error(err, "certificate watch error") + cw.log.Error(err, "certificate watch error") } } } +// updateCachedCertificate checks if the new certificate differs from the cache, +// updates it and returns the result if it was updated or not +func (cw *CertWatcher) updateCachedCertificate(cert *tls.Certificate, keyPEMBlock []byte) bool { + cw.Lock() + defer cw.Unlock() + + if cw.currentCert != nil && + bytes.Equal(cw.currentCert.Certificate[0], cert.Certificate[0]) && + bytes.Equal(cw.cachedKeyPEMBlock, keyPEMBlock) { + cw.log.V(7).Info("certificate already cached") + return false + } + cw.currentCert = cert + cw.cachedKeyPEMBlock = keyPEMBlock + return true +} + // ReadCertificate reads the certificate and key files from disk, parses them, -// and updates the current certificate on the watcher. If a callback is set, it +// and updates the current certificate on the watcher if updated. If a callback is set, it // is invoked with the new certificate. func (cw *CertWatcher) ReadCertificate() error { metrics.ReadCertificateTotal.Inc() - cert, err := tls.LoadX509KeyPair(cw.certPath, cw.keyPath) + certPEMBlock, err := os.ReadFile(cw.certPath) + if err != nil { + metrics.ReadCertificateErrors.Inc() + return err + } + keyPEMBlock, err := os.ReadFile(cw.keyPath) if err != nil { metrics.ReadCertificateErrors.Inc() return err } - cw.Lock() - cw.currentCert = &cert - cw.Unlock() + cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + metrics.ReadCertificateErrors.Inc() + return err + } + + if !cw.updateCachedCertificate(&cert, keyPEMBlock) { + return nil + } - log.Info("Updated current TLS certificate") + cw.log.Info("Updated current TLS certificate") // If a callback is registered, invoke it with the new certificate. cw.RLock() @@ -173,32 +226,26 @@ func (cw *CertWatcher) ReadCertificate() error { func (cw *CertWatcher) handleEvent(event fsnotify.Event) { // Only care about events which may modify the contents of the file. - if !(isWrite(event) || isRemove(event) || isCreate(event)) { - return - } - - log.V(1).Info("certificate event", "event", event) - - // If the file was removed, re-add the watch. - if isRemove(event) { + switch { + case event.Op.Has(fsnotify.Write): + case event.Op.Has(fsnotify.Create): + case event.Op.Has(fsnotify.Chmod), event.Op.Has(fsnotify.Remove): + // If the file was removed or renamed, re-add the watch to the previous name if err := cw.watcher.Add(event.Name); err != nil { - log.Error(err, "error re-watching file") + cw.log.Error(err, "error re-watching file") } + default: + return } + cw.log.V(1).Info("certificate event", "event", event) if err := cw.ReadCertificate(); err != nil { - log.Error(err, "error re-reading certificate") + cw.log.Error(err, "error re-reading certificate") } } -func isWrite(event fsnotify.Event) bool { - return event.Op.Has(fsnotify.Write) -} - -func isCreate(event fsnotify.Event) bool { - return event.Op.Has(fsnotify.Create) -} - -func isRemove(event fsnotify.Event) bool { - return event.Op.Has(fsnotify.Remove) +// NeedLeaderElection indicates that the cert-manager +// does not need leader election. +func (cw *CertWatcher) NeedLeaderElection() bool { + return false } diff --git a/pkg/certwatcher/certwatcher_suite_test.go b/pkg/certwatcher/certwatcher_suite_test.go index a44a968c89..d0d9a72a62 100644 --- a/pkg/certwatcher/certwatcher_suite_test.go +++ b/pkg/certwatcher/certwatcher_suite_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) @@ -41,7 +42,7 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - for _, file := range []string{certPath, keyPath} { + for _, file := range []string{certPath, keyPath, certPath + ".new", keyPath + ".new", certPath + ".old", keyPath + ".old"} { _ = os.Remove(file) } }) diff --git a/pkg/certwatcher/certwatcher_test.go b/pkg/certwatcher/certwatcher_test.go index 7e12e42679..9737925a6b 100644 --- a/pkg/certwatcher/certwatcher_test.go +++ b/pkg/certwatcher/certwatcher_test.go @@ -34,8 +34,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus/testutil" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics" + "sigs.k8s.io/controller-runtime/pkg/manager" ) var _ = Describe("CertWatcher", func() { @@ -48,13 +50,14 @@ var _ = Describe("CertWatcher", func() { var _ = Describe("certwatcher Start", func() { var ( - ctx context.Context - ctxCancel context.CancelFunc - watcher *certwatcher.CertWatcher + ctxCancel context.CancelFunc + watcher *certwatcher.CertWatcher + startWatcher func(interval time.Duration) (done <-chan struct{}) ) BeforeEach(func() { - ctx, ctxCancel = context.WithCancel(context.Background()) + var ctx context.Context + ctx, ctxCancel = context.WithCancel(context.Background()) //nolint:forbidigo // the watcher outlives the BeforeEach err := writeCerts(certPath, keyPath, "127.0.0.1") Expect(err).ToNot(HaveOccurred()) @@ -73,32 +76,40 @@ var _ = Describe("CertWatcher", func() { watcher, err = certwatcher.New(certPath, keyPath) Expect(err).ToNot(HaveOccurred()) + + startWatcher = func(interval time.Duration) (done <-chan struct{}) { + doneCh := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(doneCh) + Expect(watcher.WithWatchInterval(interval).Start(ctx)).To(Succeed()) + }() + // wait till we read first cert + Eventually(func() error { + err := watcher.ReadCertificate() + return err + }).Should(Succeed()) + return doneCh + } }) - startWatcher := func() (done <-chan struct{}) { - doneCh := make(chan struct{}) - go func() { - defer GinkgoRecover() - defer close(doneCh) - Expect(watcher.Start(ctx)).To(Succeed()) - }() - // wait till we read first cert - Eventually(func() error { - err := watcher.ReadCertificate() - return err - }).Should(Succeed()) - return doneCh - } + It("should not require LeaderElection", func() { + leaderElectionRunnable, isLeaderElectionRunnable := any(watcher).(manager.LeaderElectionRunnable) + Expect(isLeaderElectionRunnable).To(BeTrue()) + Expect(leaderElectionRunnable.NeedLeaderElection()).To(BeFalse()) + }) It("should read the initial cert/key", func() { - doneCh := startWatcher() + // This test verifies the initial read succeeded. So interval doesn't matter. + doneCh := startWatcher(10 * time.Second) ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) }) It("should reload currentCert when changed", func() { - doneCh := startWatcher() + // This test verifies fsnotify detects the cert change. So interval doesn't matter. + doneCh := startWatcher(10 * time.Second) called := atomic.Int64{} watcher.RegisterCallback(func(crt tls.Certificate) { called.Add(1) @@ -113,7 +124,38 @@ var _ = Describe("CertWatcher", func() { Eventually(func() bool { secondcert, _ := watcher.GetCertificate(nil) first := firstcert.PrivateKey.(*rsa.PrivateKey) - return first.Equal(secondcert.PrivateKey) + return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber + }).ShouldNot(BeTrue()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + Expect(called.Load()).To(BeNumerically(">=", 1)) + }) + + It("should reload currentCert when changed with rename", func() { + // This test verifies fsnotify detects the cert change. So interval doesn't matter. + doneCh := startWatcher(10 * time.Second) + called := atomic.Int64{} + watcher.RegisterCallback(func(crt tls.Certificate) { + called.Add(1) + Expect(crt.Certificate).ToNot(BeEmpty()) + }) + + firstcert, _ := watcher.GetCertificate(nil) + + err := writeCerts(certPath+".new", keyPath+".new", "192.168.0.2") + Expect(err).ToNot(HaveOccurred()) + + Expect(os.Link(certPath, certPath+".old")).To(Succeed()) + Expect(os.Rename(certPath+".new", certPath)).To(Succeed()) + + Expect(os.Link(keyPath, keyPath+".old")).To(Succeed()) + Expect(os.Rename(keyPath+".new", keyPath)).To(Succeed()) + + Eventually(func() bool { + secondcert, _ := watcher.GetCertificate(nil) + first := firstcert.PrivateKey.(*rsa.PrivateKey) + return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber }).ShouldNot(BeTrue()) ctxCancel() @@ -121,6 +163,34 @@ var _ = Describe("CertWatcher", func() { Expect(called.Load()).To(BeNumerically(">=", 1)) }) + It("should reload currentCert after move out", func() { + // This test verifies poll works, so we'll use 1s as interval (fsnotify doesn't detect this change). + doneCh := startWatcher(1 * time.Second) + called := atomic.Int64{} + watcher.RegisterCallback(func(crt tls.Certificate) { + called.Add(1) + Expect(crt.Certificate).ToNot(BeEmpty()) + }) + + firstcert, _ := watcher.GetCertificate(nil) + + Expect(os.Rename(certPath, certPath+".old")).To(Succeed()) + Expect(os.Rename(keyPath, keyPath+".old")).To(Succeed()) + + err := writeCerts(certPath, keyPath, "192.168.0.3") + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + secondcert, _ := watcher.GetCertificate(nil) + first := firstcert.PrivateKey.(*rsa.PrivateKey) + return first.Equal(secondcert.PrivateKey) || firstcert.Leaf.SerialNumber == secondcert.Leaf.SerialNumber + }, "10s", "1s").ShouldNot(BeTrue()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + Expect(called.Load()).To(BeNumerically(">=", 1)) + }) + Context("prometheus metric read_certificate_total", func() { var readCertificateTotalBefore float64 var readCertificateErrorsBefore float64 @@ -131,12 +201,13 @@ var _ = Describe("CertWatcher", func() { }) It("should get updated on successful certificate read", func() { - doneCh := startWatcher() + // This test verifies fsnotify, so interval doesn't matter. + doneCh := startWatcher(10 * time.Second) Eventually(func() error { readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) - if readCertificateTotalAfter != readCertificateTotalBefore+1.0 { - return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + if readCertificateTotalAfter < readCertificateTotalBefore+1.0 { + return fmt.Errorf("metric read certificate total expected at least: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) } return nil }, "4s").Should(Succeed()) @@ -146,12 +217,13 @@ var _ = Describe("CertWatcher", func() { }) It("should get updated on read certificate errors", func() { - doneCh := startWatcher() + // This test works with fsnotify, so interval doesn't matter. + doneCh := startWatcher(10 * time.Second) Eventually(func() error { readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) - if readCertificateTotalAfter != readCertificateTotalBefore+1.0 { - return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + if readCertificateTotalAfter < readCertificateTotalBefore+1.0 { + return fmt.Errorf("metric read certificate total expected at least: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) } readCertificateTotalBefore = readCertificateTotalAfter return nil @@ -159,17 +231,18 @@ var _ = Describe("CertWatcher", func() { Expect(os.Remove(keyPath)).To(Succeed()) + // Note, we are checking two errors here, because os.Remove generates two fsnotify events: Chmod + Remove Eventually(func() error { readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) - if readCertificateTotalAfter != readCertificateTotalBefore+1.0 { - return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + if readCertificateTotalAfter < readCertificateTotalBefore+2.0 { + return fmt.Errorf("metric read certificate total expected at least: %v and got: %v", readCertificateTotalBefore+2.0, readCertificateTotalAfter) } return nil }, "4s").Should(Succeed()) Eventually(func() error { readCertificateErrorsAfter := testutil.ToFloat64(metrics.ReadCertificateErrors) - if readCertificateErrorsAfter != readCertificateErrorsBefore+1.0 { - return fmt.Errorf("metric read certificate errors expected: %v and got: %v", readCertificateErrorsBefore+1.0, readCertificateErrorsAfter) + if readCertificateErrorsAfter < readCertificateErrorsBefore+2.0 { + return fmt.Errorf("metric read certificate errors expected at least: %v and got: %v", readCertificateErrorsBefore+2.0, readCertificateErrorsAfter) } return nil }, "4s").Should(Succeed()) diff --git a/pkg/certwatcher/example_test.go b/pkg/certwatcher/example_test.go index e322aeebfc..e85b2403cb 100644 --- a/pkg/certwatcher/example_test.go +++ b/pkg/certwatcher/example_test.go @@ -39,7 +39,7 @@ func Example() { panic(err) } - // Start goroutine with certwatcher running fsnotify against supplied certdir + // Start goroutine with certwatcher running against supplied cert go func() { if err := watcher.Start(ctx); err != nil { panic(err) diff --git a/pkg/certwatcher/metrics/metrics.go b/pkg/certwatcher/metrics/metrics.go index 05869eff03..f128abbcf0 100644 --- a/pkg/certwatcher/metrics/metrics.go +++ b/pkg/certwatcher/metrics/metrics.go @@ -18,6 +18,7 @@ package metrics import ( "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) diff --git a/pkg/client/apiutil/apimachinery.go b/pkg/client/apiutil/apimachinery.go index 6a1bfb546e..b132cb2d4d 100644 --- a/pkg/client/apiutil/apimachinery.go +++ b/pkg/client/apiutil/apimachinery.go @@ -31,11 +31,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" ) var ( @@ -60,25 +58,6 @@ func AddToProtobufScheme(addToScheme func(*runtime.Scheme) error) error { return addToScheme(protobufScheme) } -// NewDiscoveryRESTMapper constructs a new RESTMapper based on discovery -// information fetched by a new client with the given config. -func NewDiscoveryRESTMapper(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { - if httpClient == nil { - return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client") - } - - // Get a mapper - dc, err := discovery.NewDiscoveryClientForConfigAndClient(c, httpClient) - if err != nil { - return nil, err - } - gr, err := restmapper.GetAPIGroupResources(dc) - if err != nil { - return nil, err - } - return restmapper.NewDiscoveryRESTMapper(gr), nil -} - // IsObjectNamespaced returns true if the object is namespace scoped. // For unstructured objects the gvk is found from the object itself. func IsObjectNamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper meta.RESTMapper) (bool, error) { @@ -93,7 +72,10 @@ func IsObjectNamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper m // IsGVKNamespaced returns true if the object having the provided // GVK is namespace scoped. func IsGVKNamespaced(gvk schema.GroupVersionKind, restmapper meta.RESTMapper) (bool, error) { - restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}) + // Fetch the RESTMapping using the complete GVK. If we exclude the Version, the Version set + // will be populated using the cached Group if available. This can lead to failures updating + // the cache with new Versions of CRDs registered at runtime. + restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}, gvk.Version) if err != nil { return false, fmt.Errorf("failed to get restmapping: %w", err) } @@ -179,15 +161,27 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi // RESTClientForGVK constructs a new rest.Interface capable of accessing the resource associated // with the given GroupVersionKind. The REST client will be configured to use the negotiated serializer from // baseConfig, if set, otherwise a default serializer will be set. -func RESTClientForGVK(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory, httpClient *http.Client) (rest.Interface, error) { +func RESTClientForGVK( + gvk schema.GroupVersionKind, + forceDisableProtoBuf bool, + isUnstructured bool, + baseConfig *rest.Config, + codecs serializer.CodecFactory, + httpClient *http.Client, +) (rest.Interface, error) { if httpClient == nil { return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client") } - return rest.RESTClientForConfigAndClient(createRestConfig(gvk, isUnstructured, baseConfig, codecs), httpClient) + return rest.RESTClientForConfigAndClient(createRestConfig(gvk, forceDisableProtoBuf, isUnstructured, baseConfig, codecs), httpClient) } // createRestConfig copies the base config and updates needed fields for a new rest config. -func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory) *rest.Config { +func createRestConfig(gvk schema.GroupVersionKind, + forceDisableProtoBuf bool, + isUnstructured bool, + baseConfig *rest.Config, + codecs serializer.CodecFactory, +) *rest.Config { gv := gvk.GroupVersion() cfg := rest.CopyConfig(baseConfig) @@ -201,7 +195,7 @@ func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConf cfg.UserAgent = rest.DefaultKubernetesUserAgent() } // TODO(FillZpp): In the long run, we want to check discovery or something to make sure that this is actually true. - if cfg.ContentType == "" && !isUnstructured { + if cfg.ContentType == "" && !forceDisableProtoBuf { protobufSchemeLock.RLock() if protobufScheme.Recognizes(gvk) { cfg.ContentType = runtime.ContentTypeProtobuf diff --git a/pkg/client/apiutil/apimachinery_test.go b/pkg/client/apiutil/apimachinery_test.go new file mode 100644 index 0000000000..122c5cc542 --- /dev/null +++ b/pkg/client/apiutil/apimachinery_test.go @@ -0,0 +1,191 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil_test + +import ( + "context" + "strconv" + "testing" + + gmg "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +func TestApiMachinery(t *testing.T) { + for _, aggregatedDiscovery := range []bool{true, false} { + t.Run("aggregatedDiscovery="+strconv.FormatBool(aggregatedDiscovery), func(t *testing.T) { + restCfg := setupEnvtest(t, !aggregatedDiscovery) + + // Details of the GVK registered at initialization. + initialGvk := metav1.GroupVersionKind{ + Group: "crew.example.com", + Version: "v1", + Kind: "Driver", + } + + // A set of GVKs to register at runtime with varying properties. + runtimeGvks := []struct { + name string + gvk metav1.GroupVersionKind + plural string + }{ + { + name: "new Kind and Version added to existing Group", + gvk: metav1.GroupVersionKind{ + Group: "crew.example.com", + Version: "v1alpha1", + Kind: "Passenger", + }, + plural: "passengers", + }, + { + name: "new Kind added to existing Group and Version", + gvk: metav1.GroupVersionKind{ + Group: "crew.example.com", + Version: "v1", + Kind: "Garage", + }, + plural: "garages", + }, + { + name: "new GVK", + gvk: metav1.GroupVersionKind{ + Group: "inventory.example.com", + Version: "v1", + Kind: "Taxi", + }, + plural: "taxis", + }, + } + + t.Run("IsGVKNamespaced should report scope for GVK registered at initialization", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Query the scope of a GVK that was registered at initialization. + scope, err := apiutil.IsGVKNamespaced( + schema.GroupVersionKind(initialGvk), + lazyRestMapper, + ) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(scope).To(gmg.BeTrue()) + }) + + for _, runtimeGvk := range runtimeGvks { + t.Run("IsGVKNamespaced should report scope for "+runtimeGvk.name, func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + c, err := client.New(restCfg, client.Options{Scheme: s}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Run a valid query to initialize cache. + scope, err := apiutil.IsGVKNamespaced( + schema.GroupVersionKind(initialGvk), + lazyRestMapper, + ) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(scope).To(gmg.BeTrue()) + + // Register a new CRD at runtime. + crd := newCRD(t.Context(), g, c, runtimeGvk.gvk.Group, runtimeGvk.gvk.Kind, runtimeGvk.plural) + version := crd.Spec.Versions[0] + version.Name = runtimeGvk.gvk.Version + version.Storage = true + version.Served = true + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{version} + crd.Spec.Scope = apiextensionsv1.NamespaceScoped + + g.Expect(c.Create(t.Context(), crd)).To(gmg.Succeed()) + t.Cleanup(func() { + g.Expect(c.Delete(context.Background(), crd)).To(gmg.Succeed()) //nolint:forbidigo //t.Context is cancelled in t.Cleanup + }) + + // Wait until the CRD is registered. + g.Eventually(func(g gmg.Gomega) { + isRegistered, err := isCrdRegistered(restCfg, runtimeGvk.gvk) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(isRegistered).To(gmg.BeTrue()) + }).Should(gmg.Succeed(), "GVK should be available") + + // Query the scope of the GVK registered at runtime. + scope, err = apiutil.IsGVKNamespaced( + schema.GroupVersionKind(runtimeGvk.gvk), + lazyRestMapper, + ) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(scope).To(gmg.BeTrue()) + }) + } + }) + } +} + +// Check if a slice of APIResource contains a given Kind. +func kindInAPIResources(resources *metav1.APIResourceList, kind string) bool { + for _, res := range resources.APIResources { + if res.Kind == kind { + return true + } + } + return false +} + +// Check if a CRD has registered with the API server using a DiscoveryClient. +func isCrdRegistered(cfg *rest.Config, gvk metav1.GroupVersionKind) (bool, error) { + discHTTP, err := rest.HTTPClientFor(cfg) + if err != nil { + return false, err + } + + discClient, err := discovery.NewDiscoveryClientForConfigAndClient(cfg, discHTTP) + if err != nil { + return false, err + } + + resources, err := discClient.ServerResourcesForGroupVersion(gvk.Group + "/" + gvk.Version) + if err != nil { + return false, err + } + + return kindInAPIResources(resources, gvk.Kind), nil +} diff --git a/pkg/client/apiutil/errors.go b/pkg/client/apiutil/errors.go new file mode 100644 index 0000000000..c216c49d2a --- /dev/null +++ b/pkg/client/apiutil/errors.go @@ -0,0 +1,54 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "fmt" + "sort" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// ErrResourceDiscoveryFailed is returned if the RESTMapper cannot discover supported resources for some GroupVersions. +// It wraps the errors encountered, except "NotFound" errors are replaced with meta.NoResourceMatchError, for +// backwards compatibility with code that uses meta.IsNoMatchError() to check for unsupported APIs. +type ErrResourceDiscoveryFailed map[schema.GroupVersion]error + +// Error implements the error interface. +func (e *ErrResourceDiscoveryFailed) Error() string { + subErrors := []string{} + for k, v := range *e { + subErrors = append(subErrors, fmt.Sprintf("%s: %v", k, v)) + } + sort.Strings(subErrors) + return fmt.Sprintf("unable to retrieve the complete list of server APIs: %s", strings.Join(subErrors, ", ")) +} + +func (e *ErrResourceDiscoveryFailed) Unwrap() []error { + subErrors := []error{} + for gv, err := range *e { + if apierrors.IsNotFound(err) { + err = &meta.NoResourceMatchError{PartialResource: gv.WithResource("")} + } + subErrors = append(subErrors, err) + } + return subErrors +} diff --git a/pkg/client/apiutil/restmapper.go b/pkg/client/apiutil/restmapper.go index e0ff72dc13..7a7a0d1145 100644 --- a/pkg/client/apiutil/restmapper.go +++ b/pkg/client/apiutil/restmapper.go @@ -21,12 +21,14 @@ import ( "net/http" "sync" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" + "k8s.io/utils/ptr" ) // NewDynamicRESTMapper returns a dynamic RESTMapper for cfg. The dynamic @@ -40,6 +42,7 @@ func NewDynamicRESTMapper(cfg *rest.Config, httpClient *http.Client) (meta.RESTM if err != nil { return nil, err } + return &mapper{ mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), client: client, @@ -52,11 +55,15 @@ func NewDynamicRESTMapper(cfg *rest.Config, httpClient *http.Client) (meta.RESTM // client for discovery information to do REST mappings. type mapper struct { mapper meta.RESTMapper - client *discovery.DiscoveryClient + client discovery.AggregatedDiscoveryInterface knownGroups map[string]*restmapper.APIGroupResources apiGroups map[string]*metav1.APIGroup + initialDiscoveryDone bool + // mutex to provide thread-safe mapper reloading. + // It protects all fields in the mapper as well as methods + // that have the `Locked` suffix. mu sync.RWMutex } @@ -158,44 +165,78 @@ func (m *mapper) addKnownGroupAndReload(groupName string, versions ...string) er versions = nil } + m.mu.Lock() + defer m.mu.Unlock() // If no specific versions are set by user, we will scan all available ones for the API group. // This operation requires 2 requests: /api and /apis, but only once. For all subsequent calls // this data will be taken from cache. - if len(versions) == 0 { - apiGroup, err := m.findAPIGroupByName(groupName) + // + // We always run this once, because if the server supports aggregated discovery, this will + // load everything with two api calls which we assume is overall cheaper. + if len(versions) == 0 || !m.initialDiscoveryDone { + apiGroup, didAggregatedDiscovery, err := m.findAPIGroupByNameAndMaybeAggregatedDiscoveryLocked(groupName) if err != nil { return err } - for _, version := range apiGroup.Versions { - versions = append(versions, version.Version) + if apiGroup != nil && len(versions) == 0 { + for _, version := range apiGroup.Versions { + versions = append(versions, version.Version) + } } - } - m.mu.Lock() - defer m.mu.Unlock() - - // Create or fetch group resources from cache. - groupResources := &restmapper.APIGroupResources{ - Group: metav1.APIGroup{Name: groupName}, - VersionedResources: make(map[string][]metav1.APIResource), - } - if _, ok := m.knownGroups[groupName]; ok { - groupResources = m.knownGroups[groupName] + // No need to do anything further if aggregatedDiscovery is supported and we did a lookup + if didAggregatedDiscovery { + failedGroups := make(map[schema.GroupVersion]error) + for _, version := range versions { + if m.knownGroups[groupName] == nil || m.knownGroups[groupName].VersionedResources[version] == nil { + failedGroups[schema.GroupVersion{Group: groupName, Version: version}] = &meta.NoResourceMatchError{ + PartialResource: schema.GroupVersionResource{ + Group: groupName, + Version: version, + }} + } + } + if len(failedGroups) > 0 { + return ptr.To(ErrResourceDiscoveryFailed(failedGroups)) + } + return nil + } } // Update information for group resources about versioned resources. // The number of API calls is equal to the number of versions: /apis//. - groupVersionResources, err := m.fetchGroupVersionResources(groupName, versions...) + // If we encounter a missing API version (NotFound error), we will remove the group from + // the m.apiGroups and m.knownGroups caches. + // If this happens, in the next call the group will be added back to apiGroups + // and only the existing versions will be loaded in knownGroups. + groupVersionResources, err := m.fetchGroupVersionResourcesLocked(groupName, versions...) if err != nil { return fmt.Errorf("failed to get API group resources: %w", err) } - for version, resources := range groupVersionResources { - groupResources.VersionedResources[version.Version] = resources.APIResources - } + m.addGroupVersionResourcesToCacheAndReloadLocked(groupVersionResources) + return nil +} + +// addGroupVersionResourcesToCacheAndReloadLocked does what the name suggests. The mutex must be held when +// calling it. +func (m *mapper) addGroupVersionResourcesToCacheAndReloadLocked(gvr map[schema.GroupVersion]*metav1.APIResourceList) { // Update information for group resources about the API group by adding new versions. - // Ignore the versions that are already registered. - for _, version := range versions { + // Ignore the versions that are already registered + for groupVersion, resources := range gvr { + var groupResources *restmapper.APIGroupResources + if _, ok := m.knownGroups[groupVersion.Group]; ok { + groupResources = m.knownGroups[groupVersion.Group] + } else { + groupResources = &restmapper.APIGroupResources{ + Group: metav1.APIGroup{Name: groupVersion.Group}, + VersionedResources: make(map[string][]metav1.APIResource), + } + } + + version := groupVersion.Version + + groupResources.VersionedResources[version] = resources.APIResources found := false for _, v := range groupResources.Group.Versions { if v.Version == version { @@ -205,70 +246,70 @@ func (m *mapper) addKnownGroupAndReload(groupName string, versions ...string) er } if !found { - groupResources.Group.Versions = append(groupResources.Group.Versions, metav1.GroupVersionForDiscovery{ - GroupVersion: metav1.GroupVersion{Group: groupName, Version: version}.String(), + gv := metav1.GroupVersionForDiscovery{ + GroupVersion: metav1.GroupVersion{Group: groupVersion.Group, Version: version}.String(), Version: version, - }) + } + + // Prepend if preferred version, else append. The upstream DiscoveryRestMappper assumes + // the first version is the preferred one: https://github.com/kubernetes/kubernetes/blob/ef54ac803b712137871c1a1f8d635d50e69ffa6c/staging/src/k8s.io/apimachinery/pkg/api/meta/restmapper.go#L458-L461 + if group, ok := m.apiGroups[groupVersion.Group]; ok && group.PreferredVersion.Version == version { + groupResources.Group.Versions = append([]metav1.GroupVersionForDiscovery{gv}, groupResources.Group.Versions...) + } else { + groupResources.Group.Versions = append(groupResources.Group.Versions, gv) + } } - } - // Update data in the cache. - m.knownGroups[groupName] = groupResources + // Update data in the cache. + m.knownGroups[groupVersion.Group] = groupResources + } - // Finally, update the group with received information and regenerate the mapper. + // Finally, reload the mapper. updatedGroupResources := make([]*restmapper.APIGroupResources, 0, len(m.knownGroups)) for _, agr := range m.knownGroups { updatedGroupResources = append(updatedGroupResources, agr) } m.mapper = restmapper.NewDiscoveryRESTMapper(updatedGroupResources) - return nil } -// findAPIGroupByNameLocked returns API group by its name. -func (m *mapper) findAPIGroupByName(groupName string) (*metav1.APIGroup, error) { - // Looking in the cache first. - { - m.mu.RLock() - group, ok := m.apiGroups[groupName] - m.mu.RUnlock() - if ok { - return group, nil - } +// findAPIGroupByNameAndMaybeAggregatedDiscoveryLocked tries to find the passed apiGroup. +// If the server supports aggregated discovery, it will always perform that. +func (m *mapper) findAPIGroupByNameAndMaybeAggregatedDiscoveryLocked(groupName string) (_ *metav1.APIGroup, didAggregatedDiscovery bool, _ error) { + // Looking in the cache first + group, ok := m.apiGroups[groupName] + if ok { + return group, false, nil } // Update the cache if nothing was found. - apiGroups, err := m.client.ServerGroups() + apiGroups, maybeResources, _, err := m.client.GroupsAndMaybeResources() if err != nil { - return nil, fmt.Errorf("failed to get server groups: %w", err) + return nil, false, fmt.Errorf("failed to get server groups: %w", err) } if len(apiGroups.Groups) == 0 { - return nil, fmt.Errorf("received an empty API groups list") + return nil, false, fmt.Errorf("received an empty API groups list") } - m.mu.Lock() + m.initialDiscoveryDone = true for i := range apiGroups.Groups { group := &apiGroups.Groups[i] m.apiGroups[group.Name] = group } - m.mu.Unlock() - - // Looking in the cache again. - { - m.mu.RLock() - group, ok := m.apiGroups[groupName] - m.mu.RUnlock() - if ok { - return group, nil - } + if len(maybeResources) > 0 { + didAggregatedDiscovery = true + m.addGroupVersionResourcesToCacheAndReloadLocked(maybeResources) } - // If there is still nothing, return an error. - return nil, fmt.Errorf("failed to find API group %q", groupName) + // Looking in the cache again. + // Don't return an error here if the API group is not present. + // The reloaded RESTMapper will take care of returning a NoMatchError. + return m.apiGroups[groupName], didAggregatedDiscovery, nil } -// fetchGroupVersionResources fetches the resources for the specified group and its versions. -func (m *mapper) fetchGroupVersionResources(groupName string, versions ...string) (map[schema.GroupVersion]*metav1.APIResourceList, error) { +// fetchGroupVersionResourcesLocked fetches the resources for the specified group and its versions. +// This method might modify the cache so it needs to be called under the lock. +func (m *mapper) fetchGroupVersionResourcesLocked(groupName string, versions ...string) (map[schema.GroupVersion]*metav1.APIResourceList, error) { groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) failedGroups := make(map[schema.GroupVersion]error) @@ -276,9 +317,20 @@ func (m *mapper) fetchGroupVersionResources(groupName string, versions ...string groupVersion := schema.GroupVersion{Group: groupName, Version: version} apiResourceList, err := m.client.ServerResourcesForGroupVersion(groupVersion.String()) - if err != nil { + if apierrors.IsNotFound(err) { + // If the version is not found, we remove the group from the cache + // so it gets refreshed on the next call. + if m.isAPIGroupCachedLocked(groupVersion) { + delete(m.apiGroups, groupName) + } + if m.isGroupVersionCachedLocked(groupVersion) { + delete(m.knownGroups, groupName) + } + continue + } else if err != nil { failedGroups[groupVersion] = err } + if apiResourceList != nil { // even in case of error, some fallback might have been returned. groupVersionResources[groupVersion] = apiResourceList @@ -286,8 +338,35 @@ func (m *mapper) fetchGroupVersionResources(groupName string, versions ...string } if len(failedGroups) > 0 { - return nil, &discovery.ErrGroupDiscoveryFailed{Groups: failedGroups} + err := ErrResourceDiscoveryFailed(failedGroups) + return nil, &err } return groupVersionResources, nil } + +// isGroupVersionCachedLocked checks if a version for a group is cached in the known groups cache. +func (m *mapper) isGroupVersionCachedLocked(gv schema.GroupVersion) bool { + if cachedGroup, ok := m.knownGroups[gv.Group]; ok { + _, cached := cachedGroup.VersionedResources[gv.Version] + return cached + } + + return false +} + +// isAPIGroupCachedLocked checks if a version for a group is cached in the api groups cache. +func (m *mapper) isAPIGroupCachedLocked(gv schema.GroupVersion) bool { + cachedGroup, ok := m.apiGroups[gv.Group] + if !ok { + return false + } + + for _, version := range cachedGroup.Versions { + if version.Version == gv.Version { + return true + } + } + + return false +} diff --git a/pkg/client/apiutil/restmapper_test.go b/pkg/client/apiutil/restmapper_test.go index 265313be7e..51807f12de 100644 --- a/pkg/client/apiutil/restmapper_test.go +++ b/pkg/client/apiutil/restmapper_test.go @@ -18,17 +18,26 @@ package apiutil_test import ( "context" + "fmt" "net/http" + "strconv" + "sync" "testing" _ "github.com/onsi/ginkgo/v2" gmg "github.com/onsi/gomega" + "github.com/onsi/gomega/format" + gomegatypes "github.com/onsi/gomega/types" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -61,451 +70,764 @@ func (crt *countingRoundTripper) Reset() { crt.requestCount = 0 } -func setupEnvtest(t *testing.T) (*rest.Config, func(t *testing.T)) { +func setupEnvtest(t *testing.T, disableAggregatedDiscovery bool) *rest.Config { t.Log("Setup envtest") g := gmg.NewWithT(t) testEnv := &envtest.Environment{ CRDDirectoryPaths: []string{"testdata"}, } + if disableAggregatedDiscovery { + testEnv.DownloadBinaryAssets = true + testEnv.DownloadBinaryAssetsVersion = "v1.28.0" + binaryAssetsDirectory, err := envtest.SetupEnvtestDefaultBinaryAssetsDirectory() + g.Expect(err).ToNot(gmg.HaveOccurred()) + testEnv.BinaryAssetsDirectory = binaryAssetsDirectory + testEnv.ControlPlane.GetAPIServer().Configure().Append("feature-gates", "AggregatedDiscoveryEndpoint=false") + } cfg, err := testEnv.Start() g.Expect(err).NotTo(gmg.HaveOccurred()) g.Expect(cfg).NotTo(gmg.BeNil()) - teardownFunc := func(t *testing.T) { + t.Cleanup(func() { t.Log("Stop envtest") g.Expect(testEnv.Stop()).To(gmg.Succeed()) - } + }) - return cfg, teardownFunc + return cfg } func TestLazyRestMapperProvider(t *testing.T) { - restCfg, tearDownFn := setupEnvtest(t) - defer tearDownFn(t) - - t.Run("LazyRESTMapper should fetch data based on the request", func(t *testing.T) { - g := gmg.NewWithT(t) - - // For each new group it performs just one request to the API server: - // GET https://host/apis// - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - // There are no requests before any call - g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) - - mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "v1") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) - - mappings, err := lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "v1") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mappings).To(gmg.HaveLen(1)) - g.Expect(mappings[0].GroupVersionKind.Kind).To(gmg.Equal("pod")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "ingresses"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(kind.Kind).To(gmg.Equal("Ingress")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) - - kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "tokenreviews"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(kinds).To(gmg.HaveLen(1)) - g.Expect(kinds[0].Kind).To(gmg.Equal("TokenReview")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) - - resource, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "priorityclasses"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(resource.Resource).To(gmg.Equal("priorityclasses")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) - - resources, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "poddisruptionbudgets"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(resources).To(gmg.HaveLen(1)) - g.Expect(resources[0].Resource).To(gmg.Equal("poddisruptionbudgets")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) - }) - - t.Run("LazyRESTMapper should cache fetched data and doesn't perform any additional requests", func(t *testing.T) { - g := gmg.NewWithT(t) - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) - - mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) - - // Data taken from cache - there are no more additional requests. - - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) - - kind, err := lazyRestMapper.KindFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(kind.Kind).To(gmg.Equal("Deployment")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) - - resource, err := lazyRestMapper.ResourceFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(resource.Resource).To(gmg.Equal("deployments")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) - }) - - t.Run("LazyRESTMapper should work correctly with empty versions list", func(t *testing.T) { - g := gmg.NewWithT(t) - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) - - // crew.example.com has 2 versions: v1 and v2 - - // If no versions were provided by user, we fetch all of them. - // Here we expect 4 calls. - // To initialize: - // #1: GET https://host/api - // #2: GET https://host/apis - // Then, for each version it performs one request to the API server: - // #3: GET https://host/apis/crew.example.com/v1 - // #4: GET https://host/apis/crew.example.com/v2 - mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) - - // All subsequent calls won't send requests to the server. - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) - }) - - t.Run("LazyRESTMapper should work correctly with multiple API group versions", func(t *testing.T) { - g := gmg.NewWithT(t) - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) - - // We explicitly ask for 2 versions: v1 and v2. - // For each version it performs one request to the API server: - // #1: GET https://host/apis/crew.example.com/v1 - // #2: GET https://host/apis/crew.example.com/v2 - mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - // All subsequent calls won't send requests to the server as everything is stored in the cache. - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - }) - - t.Run("LazyRESTMapper should work correctly with different API group versions", func(t *testing.T) { - g := gmg.NewWithT(t) - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) - - // Now we want resources for crew.example.com/v1 version only. - // Here we expect 1 call: - // #1: GET https://host/apis/crew.example.com/v1 - mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) - - // Get additional resources from v2. - // It sends another request: - // #2: GET https://host/apis/crew.example.com/v2 - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v2") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - // No subsequent calls require additional API requests. - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - }) - - t.Run("LazyRESTMapper should return an error if the group doesn't exist", func(t *testing.T) { - g := gmg.NewWithT(t) - - // After initialization for each invalid group the mapper performs just 1 request to the API server. - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "INVALID1"}, "v1") - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) - - _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "INVALID2"}, "v1") - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "INVALID3", Version: "v1"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) - - _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "INVALID4", Version: "v1"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) - - _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "INVALID5", Version: "v1"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) - - _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "INVALID6", Version: "v1"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) - }) - - t.Run("LazyRESTMapper should return an error if a resource doesn't exist", func(t *testing.T) { - g := gmg.NewWithT(t) - - // For each invalid resource the mapper performs just 1 request to the API server. - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "INVALID"}, "v1") - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) - - _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "INVALID"}, "v1") - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "INVALID"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) - - _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "INVALID"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) - - _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "INVALID"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) - - _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "INVALID"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) - }) - - t.Run("LazyRESTMapper should return an error if the version doesn't exist", func(t *testing.T) { - g := gmg.NewWithT(t) - - // After initialization, for each invalid resource mapper performs 1 requests to the API server. - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "INVALID") - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) - - _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "INVALID") - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) - - _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "INVALID", Resource: "ingresses"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + for _, aggregatedDiscovery := range []bool{true, false} { + t.Run("aggregatedDiscovery="+strconv.FormatBool(aggregatedDiscovery), func(t *testing.T) { + restCfg := setupEnvtest(t, !aggregatedDiscovery) + + t.Run("LazyRESTMapper should fetch data based on the request", func(t *testing.T) { + g := gmg.NewWithT(t) + + // For each new group it performs just one request to the API server: + // GET https://host/apis// + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + mappings, err := lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mappings).To(gmg.HaveLen(1)) + g.Expect(mappings[0].GroupVersionKind.Kind).To(gmg.Equal("pod")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "ingresses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Kind).To(gmg.Equal("Ingress")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "tokenreviews"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kinds).To(gmg.HaveLen(1)) + g.Expect(kinds[0].Kind).To(gmg.Equal("TokenReview")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + resource, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "priorityclasses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resource.Resource).To(gmg.Equal("priorityclasses")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + resources, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "poddisruptionbudgets"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resources).To(gmg.HaveLen(1)) + g.Expect(resources[0].Resource).To(gmg.Equal("poddisruptionbudgets")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should cache fetched data and doesn't perform any additional requests", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // Data taken from cache - there are no more additional requests. + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + kind, err := lazyRestMapper.KindFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Kind).To(gmg.Equal("Deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + resource, err := lazyRestMapper.ResourceFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resource.Resource).To(gmg.Equal("deployments")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should work correctly with empty versions list", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // crew.example.com has 2 versions: v1 and v2 + + // If no versions were provided by user, we fetch all of them. + // Here we expect 4 calls. + // To initialize: + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each version it performs one request to the API server: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // All subsequent calls won't send requests to the server. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should work correctly with multiple API group versions", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // We explicitly ask for 2 versions: v1 and v2. + // For each version it performs one request to the API server: + // #1: GET https://host/apis/crew.example.com/v1 + // #2: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // All subsequent calls won't send requests to the server as everything is stored in the cache. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should work correctly with different API group versions", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Now we want resources for crew.example.com/v1 version only. + // Here we expect 1 call: + // #1: GET https://host/apis/crew.example.com/v1 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // Get additional resources from v2. + // It sends another request: + // #2: GET https://host/apis/crew.example.com/v2 + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + if !aggregatedDiscovery { + expectedAPIRequestCount++ + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + // No subsequent calls require additional API requests. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + }) + + t.Run("LazyRESTMapper should return an error if the group doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // After initialization for each invalid group the mapper performs just 1 request to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // A version is specified but the group doesn't exist. + // For each group, we expect 1 call to the version-specific discovery endpoint: + // #1: GET https://host/apis// + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "INVALID1"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "INVALID2"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "INVALID3", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "INVALID4", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "INVALID5", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "INVALID6", Version: "v1", Resource: "invalid"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + + // No version is specified but the group doesn't exist. + // For each group, we expect 2 calls to discover all group versions: + // #1: GET https://host/api + // #2: GET https://host/apis + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "INVALID7"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(7)) + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "INVALID8"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(9)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "INVALID9", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(11)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "INVALID10", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(13)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "INVALID11", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(15)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "INVALID12", Resource: "invalid"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(17)) + }) + + t.Run("LazyRESTMapper should return an error if a resource doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // For each invalid resource the mapper performs just 1 request to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "INVALID"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "INVALID"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + }) + + t.Run("LazyRESTMapper should return an error if the version doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // After initialization, for each invalid resource mapper performs 1 requests to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "INVALID") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + expectedAPIRequestCount := 3 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "INVALID") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "INVALID", Resource: "ingresses"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "INVALID", Resource: "tokenreviews"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "INVALID", Resource: "priorityclasses"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "INVALID", Resource: "poddisruptionbudgets"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(meta.IsNoMatchError(err)).To(gmg.BeTrue()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + }) + + t.Run("LazyRESTMapper should work correctly if the version isn't specified", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Resource: "ingress"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Version).ToNot(gmg.BeEmpty()) + + kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Resource: "tokenreviews"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kinds).ToNot(gmg.BeEmpty()) + g.Expect(kinds[0].Version).ToNot(gmg.BeEmpty()) + + resorce, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Resource: "priorityclasses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resorce.Version).ToNot(gmg.BeEmpty()) + + resorces, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Resource: "poddisruptionbudgets"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kinds).ToNot(gmg.BeEmpty()) + g.Expect(resorces[0].Version).ToNot(gmg.BeEmpty()) + }) + + t.Run("LazyRESTMapper can fetch CRDs if they were created at runtime", func(t *testing.T) { + g := gmg.NewWithT(t) + + // To fetch all versions mapper does 2 requests: + // GET https://host/api + // GET https://host/apis + // Then, for each version it performs just one request to the API server as usual: + // GET https://host/apis// + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each currently registered version: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + c, err := client.New(restCfg, client.Options{Scheme: s}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Register another CRD in runtime - "riders.crew.example.com". + createNewCRD(t.Context(), g, c, "crew.example.com", "Rider", "riders") + + // Wait a bit until the CRD is registered. + g.Eventually(func() error { + _, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) + return err + }).Should(gmg.Succeed()) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each currently registered version: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("rider")) + }) + + t.Run("LazyRESTMapper should invalidate the group cache if a version is not found", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + c, err := client.New(restCfg, client.Options{Scheme: s}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Register a new CRD ina new group to avoid collisions when deleting versions - "taxis.inventory.example.com". + group := "inventory.example.com" + kind := "Taxi" + plural := "taxis" + crdName := plural + "." + group + // Create a CRD with two versions: v1alpha1 and v1 where both are served and + // v1 is the storage version so we can easily remove v1alpha1 later. + crd := newCRD(t.Context(), g, c, group, kind, plural) + v1alpha1 := crd.Spec.Versions[0] + v1alpha1.Name = "v1alpha1" + v1alpha1.Storage = false + v1alpha1.Served = true + v1 := crd.Spec.Versions[0] + v1.Name = "v1" + v1.Storage = true + v1.Served = true + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{v1alpha1, v1} + g.Expect(c.Create(t.Context(), crd)).To(gmg.Succeed()) + t.Cleanup(func() { + g.Expect(c.Delete(context.Background(), crd)).To(gmg.Succeed()) //nolint:forbidigo //t.Context is cancelled in t.Cleanup + }) + + // Wait until the CRD is registered. + discHTTP, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + discClient, err := discovery.NewDiscoveryClientForConfigAndClient(restCfg, discHTTP) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Eventually(func(g gmg.Gomega) { + _, err = discClient.ServerResourcesForGroupVersion(group + "/v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + }).Should(gmg.Succeed(), "v1 should be available") + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for all available versions: + // #3: GET https://host/apis/inventory.example.com/v1alpha1 + // #4: GET https://host/apis/inventory.example.com/v1 + // This should fill the cache for apiGroups and versions. + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal(kind)) + expectedAPIRequestCount := 4 + if aggregatedDiscovery { + expectedAPIRequestCount = 2 + } + g.Expect(crt.GetRequestCount()).To(gmg.Equal(expectedAPIRequestCount)) + crt.Reset() // We reset the counter to check how many additional requests are made later. + + // At this point v1alpha1 should be cached + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}, "v1alpha1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // We update the CRD to only have v1 version. + g.Expect(c.Get(t.Context(), types.NamespacedName{Name: crdName}, crd)).To(gmg.Succeed()) + for _, version := range crd.Spec.Versions { + if version.Name == "v1" { + v1 = version + break + } + } + crd.Spec.Versions = []apiextensionsv1.CustomResourceDefinitionVersion{v1} + g.Expect(c.Update(t.Context(), crd)).To(gmg.Succeed()) + + // We wait until v1alpha1 is not available anymore. + g.Eventually(func(g gmg.Gomega) { + _, err = discClient.ServerResourcesForGroupVersion(group + "/v1alpha1") + g.Expect(apierrors.IsNotFound(err)).To(gmg.BeTrue(), "v1alpha1 should not be available anymore") + }).Should(gmg.Succeed()) + + // Although v1alpha1 is not available anymore, the cache is not invalidated yet so it should return a mapping. + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}, "v1alpha1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // We request Limo, which is not in the mapper because it doesn't exist. + // This will trigger a reload of the lazy mapper cache. + // Reloading the cache will read v2 again and since it's not available anymore, it should invalidate the cache. + // #1: GET https://host/apis/inventory.example.com/v1alpha1 + // #2: GET https://host/apis/inventory.example.com/v1 + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: "Limo"}) + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + crt.Reset() + + // Now we request v1alpha1 again and it should return an error since the cache was invalidated. + // #1: GET https://host/apis/inventory.example.com/v1alpha1 + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}, "v1alpha1") + g.Expect(err).To(beNoMatchError()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + // Verify that when requesting the mapping without a version, it doesn't error + // and it returns v1. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.Resource.Version).To(gmg.Equal("v1")) + }) + + t.Run("Restmapper should consistently return the preferred version", func(t *testing.T) { + g := gmg.NewWithT(t) + + wg := sync.WaitGroup{} + wg.Add(50) + for i := 0; i < 50; i++ { + go func() { + defer wg.Done() + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + mapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + mapping, err := mapper.RESTMapping(schema.GroupKind{ + Group: "crew.example.com", + Kind: "Driver", + }) + g.Expect(err).NotTo(gmg.HaveOccurred()) + // APIServer seems to have a heuristic to prefer the higher + // version number. + g.Expect(mapping.GroupVersionKind.Version).To(gmg.Equal("v2")) + }() + } + wg.Wait() + }) + }) + } +} - _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "INVALID", Resource: "tokenreviews"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) +// createNewCRD creates a new CRD with the given group, kind, and plural and returns it. +func createNewCRD(ctx context.Context, g gmg.Gomega, c client.Client, group, kind, plural string) *apiextensionsv1.CustomResourceDefinition { + newCRD := newCRD(ctx, g, c, group, kind, plural) + g.Expect(c.Create(ctx, newCRD)).To(gmg.Succeed()) - _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "INVALID", Resource: "priorityclasses"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + return newCRD +} - _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "INVALID", Resource: "poddisruptionbudgets"}) - g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) - }) +// newCRD returns a new CRD with the given group, kind, and plural. +func newCRD(ctx context.Context, g gmg.Gomega, c client.Client, group, kind, plural string) *apiextensionsv1.CustomResourceDefinition { + crd := &apiextensionsv1.CustomResourceDefinition{} + err := c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(crd.Spec.Names.Kind).To(gmg.Equal("Driver")) + + newCRD := &apiextensionsv1.CustomResourceDefinition{} + crd.DeepCopyInto(newCRD) + newCRD.Spec.Group = group + newCRD.Name = plural + "." + group + newCRD.Spec.Names = apiextensionsv1.CustomResourceDefinitionNames{ + Kind: kind, + Plural: plural, + } + newCRD.ResourceVersion = "" - t.Run("LazyRESTMapper should work correctly if the version isn't specified", func(t *testing.T) { - g := gmg.NewWithT(t) + return newCRD +} - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) +func beNoMatchError() gomegatypes.GomegaMatcher { + return &errorMatcher{ + checkFunc: meta.IsNoMatchError, + message: "NoMatch", + } +} - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) +type errorMatcher struct { + checkFunc func(error) bool + message string +} - kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Resource: "ingress"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(kind.Version).ToNot(gmg.BeEmpty()) +func (e *errorMatcher) Match(actual interface{}) (success bool, err error) { + if actual == nil { + return false, nil + } - kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Resource: "tokenreviews"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(kinds).ToNot(gmg.BeEmpty()) - g.Expect(kinds[0].Version).ToNot(gmg.BeEmpty()) + actualErr, actualOk := actual.(error) + if !actualOk { + return false, fmt.Errorf("expected an error-type. got:\n%s", format.Object(actual, 1)) + } - resorce, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Resource: "priorityclasses"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(resorce.Version).ToNot(gmg.BeEmpty()) + return e.checkFunc(actualErr), nil +} - resorces, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Resource: "poddisruptionbudgets"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(kinds).ToNot(gmg.BeEmpty()) - g.Expect(resorces[0].Version).ToNot(gmg.BeEmpty()) - }) +func (e *errorMatcher) FailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("to be %s error", e.message)) +} - t.Run("LazyRESTMapper can fetch CRDs if they were created at runtime", func(t *testing.T) { - g := gmg.NewWithT(t) - - // To fetch all versions mapper does 2 requests: - // GET https://host/api - // GET https://host/apis - // Then, for each version it performs just one request to the API server as usual: - // GET https://host/apis// - - httpClient, err := rest.HTTPClientFor(restCfg) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - crt := newCountingRoundTripper(httpClient.Transport) - httpClient.Transport = crt - - lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - // There are no requests before any call - g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) - - // Since we don't specify what version we expect, restmapper will fetch them all and search there. - // To fetch a list of available versions - // #1: GET https://host/api - // #2: GET https://host/apis - // Then, for each currently registered version: - // #3: GET https://host/apis/crew.example.com/v1 - // #4: GET https://host/apis/crew.example.com/v2 - mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) - - s := scheme.Scheme - err = apiextensionsv1.AddToScheme(s) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - c, err := client.New(restCfg, client.Options{Scheme: s}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - - // Register another CRD in runtime - "riders.crew.example.com". - - crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(crd.Spec.Names.Kind).To(gmg.Equal("Driver")) - - newCRD := &apiextensionsv1.CustomResourceDefinition{} - crd.DeepCopyInto(newCRD) - newCRD.Name = "riders.crew.example.com" - newCRD.Spec.Names = apiextensionsv1.CustomResourceDefinitionNames{ - Kind: "Rider", - Plural: "riders", - } - newCRD.ResourceVersion = "" - - // Create the new CRD. - g.Expect(c.Create(context.TODO(), newCRD)).To(gmg.Succeed()) - - // Wait a bit until the CRD is registered. - g.Eventually(func() error { - _, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) - return err - }).Should(gmg.Succeed()) - - // Since we don't specify what version we expect, restmapper will fetch them all and search there. - // To fetch a list of available versions - // #1: GET https://host/api - // #2: GET https://host/apis - // Then, for each currently registered version: - // #3: GET https://host/apis/crew.example.com/v1 - // #4: GET https://host/apis/crew.example.com/v2 - mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) - g.Expect(err).NotTo(gmg.HaveOccurred()) - g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("rider")) - }) +func (e *errorMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return format.Message(actual, fmt.Sprintf("not to be %s error", e.message)) } diff --git a/pkg/client/apiutil/restmapper_wb_test.go b/pkg/client/apiutil/restmapper_wb_test.go new file mode 100644 index 0000000000..73c4236724 --- /dev/null +++ b/pkg/client/apiutil/restmapper_wb_test.go @@ -0,0 +1,214 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "testing" + + gmg "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/restmapper" +) + +func TestLazyRestMapper_fetchGroupVersionResourcesLocked_CacheInvalidation(t *testing.T) { + tests := []struct { + name string + groupName string + versions []string + cachedAPIGroups, expectedAPIGroups map[string]*metav1.APIGroup + cachedKnownGroups, expectedKnownGroups map[string]*restmapper.APIGroupResources + }{ + { + name: "Not found version for cached groupVersion in apiGroups and knownGroups", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v1", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v1": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{}, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{}, + }, + { + name: "Not found version for cached groupVersion only in apiGroups", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v1", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{}, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + }, + { + name: "Not found version for cached groupVersion only in knownGroups", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v2": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{}, + }, + { + name: "Not found version for non cached groupVersion", + groupName: "group1", + versions: []string{"v1", "v2"}, + cachedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + cachedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + expectedAPIGroups: map[string]*metav1.APIGroup{ + "group1": { + Name: "group1", + Versions: []metav1.GroupVersionForDiscovery{ + { + Version: "v3", + }, + }, + }, + }, + expectedKnownGroups: map[string]*restmapper.APIGroupResources{ + "group1": { + VersionedResources: map[string][]metav1.APIResource{ + "v3": { + { + Name: "resource1", + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := gmg.NewWithT(t) + m := &mapper{ + mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), + client: &fakeAggregatedDiscoveryClient{DiscoveryInterface: fake.NewSimpleClientset().Discovery()}, + apiGroups: tt.cachedAPIGroups, + knownGroups: tt.cachedKnownGroups, + } + _, err := m.fetchGroupVersionResourcesLocked(tt.groupName, tt.versions...) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(m.apiGroups).To(gmg.BeComparableTo(tt.expectedAPIGroups)) + g.Expect(m.knownGroups).To(gmg.BeComparableTo(tt.expectedKnownGroups)) + }) + } +} + +type fakeAggregatedDiscoveryClient struct { + discovery.DiscoveryInterface +} + +func (f *fakeAggregatedDiscoveryClient) GroupsAndMaybeResources() (*metav1.APIGroupList, map[schema.GroupVersion]*metav1.APIResourceList, map[schema.GroupVersion]error, error) { + groupList, err := f.DiscoveryInterface.ServerGroups() + return groupList, nil, nil, err +} diff --git a/pkg/client/applyconfigurations.go b/pkg/client/applyconfigurations.go new file mode 100644 index 0000000000..97192050f9 --- /dev/null +++ b/pkg/client/applyconfigurations.go @@ -0,0 +1,75 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" +) + +type unstructuredApplyConfiguration struct { + *unstructured.Unstructured +} + +func (u *unstructuredApplyConfiguration) IsApplyConfiguration() {} + +// ApplyConfigurationFromUnstructured creates a runtime.ApplyConfiguration from an *unstructured.Unstructured object. +// +// Do not use Unstructured objects here that were generated from API objects, as its impossible to tell +// if a zero value was explicitly set. +func ApplyConfigurationFromUnstructured(u *unstructured.Unstructured) runtime.ApplyConfiguration { + return &unstructuredApplyConfiguration{Unstructured: u} +} + +type applyconfigurationRuntimeObject struct { + runtime.ApplyConfiguration +} + +func (a *applyconfigurationRuntimeObject) GetObjectKind() schema.ObjectKind { + return a +} + +func (a *applyconfigurationRuntimeObject) GroupVersionKind() schema.GroupVersionKind { + return schema.GroupVersionKind{} +} + +func (a *applyconfigurationRuntimeObject) SetGroupVersionKind(gvk schema.GroupVersionKind) {} + +func (a *applyconfigurationRuntimeObject) DeepCopyObject() runtime.Object { + panic("applyconfigurationRuntimeObject does not support DeepCopyObject") +} + +func runtimeObjectFromApplyConfiguration(ac runtime.ApplyConfiguration) runtime.Object { + return &applyconfigurationRuntimeObject{ApplyConfiguration: ac} +} + +func gvkFromApplyConfiguration(ac applyConfiguration) (schema.GroupVersionKind, error) { + var gvk schema.GroupVersionKind + gv, err := schema.ParseGroupVersion(ptr.Deref(ac.GetAPIVersion(), "")) + if err != nil { + return gvk, fmt.Errorf("failed to parse %q as GroupVersion: %w", ptr.Deref(ac.GetAPIVersion(), ""), err) + } + gvk.Group = gv.Group + gvk.Version = gv.Version + gvk.Kind = ptr.Deref(ac.GetKind(), "") + + return gvk, nil +} diff --git a/pkg/client/client.go b/pkg/client/client.go index 21067b6f8f..39050de457 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -50,37 +50,21 @@ type Options struct { // Cache, if provided, is used to read objects from the cache. Cache *CacheOptions - // WarningHandler is used to configure the warning handler responsible for - // surfacing and handling warnings messages sent by the API server. - WarningHandler WarningHandlerOptions - // DryRun instructs the client to only perform dry run requests. DryRun *bool } -// WarningHandlerOptions are options for configuring a -// warning handler for the client which is responsible -// for surfacing API Server warnings. -type WarningHandlerOptions struct { - // SuppressWarnings decides if the warnings from the - // API server are suppressed or surfaced in the client. - SuppressWarnings bool - // AllowDuplicateLogs does not deduplicate the to-be - // logged surfaced warnings messages. See - // log.WarningHandlerOptions for considerations - // regarding deduplication - AllowDuplicateLogs bool -} - // CacheOptions are options for creating a cache-backed client. type CacheOptions struct { // Reader is a cache-backed reader that will be used to read objects from the cache. // +required Reader Reader - // DisableFor is a list of objects that should not be read from the cache. + // DisableFor is a list of objects that should never be read from the cache. + // Objects configured here always result in a live lookup. DisableFor []Object // Unstructured is a flag that indicates whether the cache-backed client should // read unstructured objects or lists from the cache. + // If false, unstructured objects will always result in a live lookup. Unstructured bool } @@ -88,11 +72,24 @@ type CacheOptions struct { type NewClientFunc func(config *rest.Config, options Options) (Client, error) // New returns a new Client using the provided config and Options. -// The returned client reads *and* writes directly from the server -// (it doesn't use object caches). It understands how to work with -// normal types (both custom resources and aggregated/built-in resources), -// as well as unstructured types. // +// By default, the client surfaces warnings returned by the server. To +// suppress warnings, set config.WarningHandlerWithContext = rest.NoWarnings{}. To +// define custom behavior, implement the rest.WarningHandlerWithContext interface. +// See [sigs.k8s.io/controller-runtime/pkg/log.KubeAPIWarningLogger] for +// an example. +// +// The client's read behavior is determined by Options.Cache. +// If either Options.Cache or Options.Cache.Reader is nil, +// the client reads directly from the API server. +// If both Options.Cache and Options.Cache.Reader are non-nil, +// the client reads from a local cache. However, specific +// resources can still be configured to bypass the cache based +// on Options.Cache.Unstructured and Options.Cache.DisableFor. +// Write operations are always performed directly on the API server. +// +// The client understands how to work with normal types (both custom resources +// and aggregated/built-in resources), as well as unstructured types. // In the case of normal types, the scheme will be used to look up the // corresponding group, version, and kind for the given type. In the // case of unstructured types, the group, version, and kind will be extracted @@ -110,18 +107,16 @@ func newClient(config *rest.Config, options Options) (*client, error) { return nil, fmt.Errorf("must provide non-nil rest.Config to client.New") } - if !options.WarningHandler.SuppressWarnings { - // surface warnings - logger := log.Log.WithName("KubeAPIWarningLogger") - // Set a WarningHandler, the default WarningHandler - // is log.KubeAPIWarningLogger with deduplication enabled. - // See log.KubeAPIWarningLoggerOptions for considerations - // regarding deduplication. - config = rest.CopyConfig(config) - config.WarningHandler = log.NewKubeAPIWarningLogger( - logger, + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + if config.WarningHandler == nil && config.WarningHandlerWithContext == nil { + // By default, we surface warnings. + config.WarningHandlerWithContext = log.NewKubeAPIWarningLogger( log.KubeAPIWarningLoggerOptions{ - Deduplicate: !options.WarningHandler.AllowDuplicateLogs, + Deduplicate: false, }, ) } @@ -156,11 +151,10 @@ func newClient(config *rest.Config, options Options) (*client, error) { mapper: options.Mapper, codecs: serializer.NewCodecFactory(options.Scheme), - structuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta), - unstructuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta), + resourceByType: make(map[cacheKey]*resourceMeta), } - rawMetaClient, err := metadata.NewForConfigAndClient(config, options.HTTPClient) + rawMetaClient, err := metadata.NewForConfigAndClient(metadata.ConfigFor(config), options.HTTPClient) if err != nil { return nil, fmt.Errorf("unable to construct metadata-only client for use as part of client: %w", err) } @@ -204,7 +198,8 @@ func newClient(config *rest.Config, options Options) (*client, error) { var _ Client = &client{} -// client is a client.Client that reads and writes directly from/to an API server. +// client is a client.Client configured to either read from a local cache or directly from the API server. +// Write operations are always performed directly on the API server. // It lazily initializes new clients at the time they are used. type client struct { typedClient typedClient @@ -333,14 +328,26 @@ func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...Pat } } +func (c *client) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + switch obj := obj.(type) { + case *unstructuredApplyConfiguration: + defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + return c.unstructuredClient.Apply(ctx, obj, opts...) + default: + return c.typedClient.Apply(ctx, obj, opts...) + } +} + // Get implements client.Client. func (c *client) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { if isUncached, err := c.shouldBypassCache(obj); err != nil { return err } else if !isUncached { + // Attempt to get from the cache. return c.cache.Get(ctx, key, obj, opts...) } + // Perform a live lookup. switch obj.(type) { case runtime.Unstructured: return c.unstructuredClient.Get(ctx, key, obj, opts...) @@ -358,9 +365,11 @@ func (c *client) List(ctx context.Context, obj ObjectList, opts ...ListOption) e if isUncached, err := c.shouldBypassCache(obj); err != nil { return err } else if !isUncached { + // Attempt to get from the cache. return c.cache.List(ctx, obj, opts...) } + // Perform a live lookup. switch x := obj.(type) { case runtime.Unstructured: return c.unstructuredClient.List(ctx, obj, opts...) @@ -505,8 +514,8 @@ func (co *SubResourceCreateOptions) ApplyOptions(opts []SubResourceCreateOption) return co } -// ApplyToSubresourceCreate applies the the configuration on the given create options. -func (co *SubResourceCreateOptions) ApplyToSubresourceCreate(o *SubResourceCreateOptions) { +// ApplyToSubResourceCreate applies the the configuration on the given create options. +func (co *SubResourceCreateOptions) ApplyToSubResourceCreate(o *SubResourceCreateOptions) { co.CreateOptions.ApplyToCreate(&co.CreateOptions) } @@ -534,6 +543,30 @@ func (po *SubResourcePatchOptions) ApplyToSubResourcePatch(o *SubResourcePatchOp } } +// SubResourceApplyOptions are the options for a subresource +// apply request. +type SubResourceApplyOptions struct { + ApplyOptions + SubResourceBody runtime.ApplyConfiguration +} + +// ApplyOpts applies the given options. +func (ao *SubResourceApplyOptions) ApplyOpts(opts []SubResourceApplyOption) *SubResourceApplyOptions { + for _, o := range opts { + o.ApplyToSubResourceApply(ao) + } + + return ao +} + +// ApplyToSubResourceApply applies the configuration on the given patch options. +func (ao *SubResourceApplyOptions) ApplyToSubResourceApply(o *SubResourceApplyOptions) { + ao.ApplyOptions.ApplyToApply(&o.ApplyOptions) + if ao.SubResourceBody != nil { + o.SubResourceBody = ao.SubResourceBody + } +} + func (sc *subResourceClient) Get(ctx context.Context, obj Object, subResource Object, opts ...SubResourceGetOption) error { switch obj.(type) { case runtime.Unstructured: @@ -585,3 +618,13 @@ func (sc *subResourceClient) Patch(ctx context.Context, obj Object, patch Patch, return sc.client.typedClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...) } } + +func (sc *subResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + switch obj := obj.(type) { + case *unstructuredApplyConfiguration: + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + return sc.client.unstructuredClient.ApplySubResource(ctx, obj, sc.subResource, opts...) + default: + return sc.client.typedClient.ApplySubResource(ctx, obj, sc.subResource, opts...) + } +} diff --git a/pkg/client/client_rest_resources.go b/pkg/client/client_rest_resources.go index 2d07879520..d75d685cbb 100644 --- a/pkg/client/client_rest_resources.go +++ b/pkg/client/client_rest_resources.go @@ -17,16 +17,17 @@ limitations under the License. package client import ( + "fmt" "net/http" "strings" "sync" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/rest" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) @@ -47,22 +48,30 @@ type clientRestResources struct { // codecs are used to create a REST client for a gvk codecs serializer.CodecFactory - // structuredResourceByType stores structured type metadata - structuredResourceByType map[schema.GroupVersionKind]*resourceMeta - // unstructuredResourceByType stores unstructured type metadata - unstructuredResourceByType map[schema.GroupVersionKind]*resourceMeta - mu sync.RWMutex + // resourceByType stores type metadata + resourceByType map[cacheKey]*resourceMeta + + mu sync.RWMutex +} + +type cacheKey struct { + gvk schema.GroupVersionKind + forceDisableProtoBuf bool } // newResource maps obj to a Kubernetes Resource and constructs a client for that Resource. // If the object is a list, the resource represents the item's type instead. -func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) { +func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, + isList bool, + forceDisableProtoBuf bool, + isUnstructured bool, +) (*resourceMeta, error) { if strings.HasSuffix(gvk.Kind, "List") && isList { // if this was a list, treat it as a request for the item's resource gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] } - client, err := apiutil.RESTClientForGVK(gvk, isUnstructured, c.config, c.codecs, c.httpClient) + client, err := apiutil.RESTClientForGVK(gvk, forceDisableProtoBuf, isUnstructured, c.config, c.codecs, c.httpClient) if err != nil { return nil, err } @@ -73,52 +82,96 @@ func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, i return &resourceMeta{Interface: client, mapping: mapping, gvk: gvk}, nil } +type applyConfiguration interface { + GetName() *string + GetNamespace() *string + GetKind() *string + GetAPIVersion() *string +} + // getResource returns the resource meta information for the given type of object. // If the object is a list, the resource represents the item's type instead. -func (c *clientRestResources) getResource(obj runtime.Object) (*resourceMeta, error) { - gvk, err := apiutil.GVKForObject(obj, c.scheme) - if err != nil { - return nil, err +func (c *clientRestResources) getResource(obj any) (*resourceMeta, error) { + var gvk schema.GroupVersionKind + var err error + var isApplyConfiguration bool + switch o := obj.(type) { + case runtime.Object: + gvk, err = apiutil.GVKForObject(o, c.scheme) + if err != nil { + return nil, err + } + case runtime.ApplyConfiguration: + ac, ok := o.(applyConfiguration) + if !ok { + return nil, fmt.Errorf("%T is a runtime.ApplyConfiguration but not an applyConfiguration", o) + } + gvk, err = gvkFromApplyConfiguration(ac) + if err != nil { + return nil, err + } + isApplyConfiguration = true + default: + return nil, fmt.Errorf("bug: %T is neither a runtime.Object nor a runtime.ApplyConfiguration", o) } _, isUnstructured := obj.(runtime.Unstructured) + forceDisableProtoBuf := isUnstructured || isApplyConfiguration // It's better to do creation work twice than to not let multiple // people make requests at once c.mu.RLock() - resourceByType := c.structuredResourceByType - if isUnstructured { - resourceByType = c.unstructuredResourceByType - } - r, known := resourceByType[gvk] + + cacheKey := cacheKey{gvk: gvk, forceDisableProtoBuf: forceDisableProtoBuf} + + r, known := c.resourceByType[cacheKey] + c.mu.RUnlock() if known { return r, nil } + var isList bool + if runtimeObject, ok := obj.(runtime.Object); ok && meta.IsListType(runtimeObject) { + isList = true + } + // Initialize a new Client c.mu.Lock() defer c.mu.Unlock() - r, err = c.newResource(gvk, meta.IsListType(obj), isUnstructured) + r, err = c.newResource(gvk, isList, forceDisableProtoBuf, isUnstructured) if err != nil { return nil, err } - resourceByType[gvk] = r + c.resourceByType[cacheKey] = r return r, err } // getObjMeta returns objMeta containing both type and object metadata and state. -func (c *clientRestResources) getObjMeta(obj runtime.Object) (*objMeta, error) { +func (c *clientRestResources) getObjMeta(obj any) (*objMeta, error) { r, err := c.getResource(obj) if err != nil { return nil, err } - m, err := meta.Accessor(obj) - if err != nil { - return nil, err + objMeta := &objMeta{resourceMeta: r} + + switch o := obj.(type) { + case runtime.Object: + m, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + objMeta.namespace = m.GetNamespace() + objMeta.name = m.GetName() + case applyConfiguration: + objMeta.namespace = ptr.Deref(o.GetNamespace(), "") + objMeta.name = ptr.Deref(o.GetName(), "") + default: + return nil, fmt.Errorf("object %T is neither a runtime.Object nor a runtime.ApplyConfiguration", obj) } - return &objMeta{resourceMeta: r, Object: m}, err + + return objMeta, nil } // resourceMeta stores state for a Kubernetes type. @@ -146,6 +199,6 @@ type objMeta struct { // resourceMeta contains type information for the object *resourceMeta - // Object contains meta data for the object instance - metav1.Object + namespace string + name string } diff --git a/pkg/client/client_suite_test.go b/pkg/client/client_suite_test.go index f3942502d3..89cab3f7ed 100644 --- a/pkg/client/client_suite_test.go +++ b/pkg/client/client_suite_test.go @@ -17,6 +17,8 @@ limitations under the License. package client_test import ( + "bytes" + "io" "testing" . "github.com/onsi/ginkgo/v2" @@ -24,6 +26,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/examples/crd/pkg" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -36,12 +39,24 @@ func TestClient(t *testing.T) { RunSpecs(t, "Client Suite") } -var testenv *envtest.Environment -var cfg *rest.Config -var clientset *kubernetes.Clientset +var ( + testenv *envtest.Environment + cfg *rest.Config + clientset *kubernetes.Clientset + + // Used by tests to inspect controller and client log messages. + log bytes.Buffer +) var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + // Forwards logs to ginkgo output, and allows tests to inspect logs. + mw := io.MultiWriter(&log, GinkgoWriter) + + // Use prefixes to help us tell the source of the log message. + // controller-runtime uses logf + logf.SetLogger(zap.New(zap.WriteTo(mw), zap.UseDevMode(true)).WithName("logf")) + // client-go logs uses klog + klog.SetLogger(zap.New(zap.WriteTo(mw), zap.UseDevMode(true)).WithName("klog")) testenv = &envtest.Environment{CRDDirectoryPaths: []string{"./testdata"}} diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index d8b3995050..021fbeb0d8 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -17,10 +17,14 @@ limitations under the License. package client_test import ( + "bufio" + "bytes" "context" "encoding/json" + "errors" "fmt" "reflect" + "strings" "sync/atomic" "time" @@ -39,10 +43,15 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + appsv1applyconfigurations "k8s.io/client-go/applyconfigurations/apps/v1" + autoscaling1applyconfigurations "k8s.io/client-go/applyconfigurations/autoscaling/v1" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" kscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/utils/pointer" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/examples/crd/pkg" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -143,7 +152,7 @@ var _ = Describe("Client", func() { var count uint64 = 0 var replicaCount int32 = 2 var ns = "default" - ctx := context.TODO() + var errNotCached *cache.ErrResourceNotCached BeforeEach(func() { atomic.AddUint64(&count, 1) @@ -202,7 +211,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) var delOptions *metav1.DeleteOptions - AfterEach(func() { + AfterEach(func(ctx SpecContext) { // Cleanup var zero int64 = 0 policy := metav1.DeletePropagationForeground @@ -223,6 +232,65 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred()) }) + Describe("WarningHandler", func() { + It("should log warnings with config.WarningHandler, if one is defined", func(ctx SpecContext) { + cache := &fakeReader{} + + testCfg := rest.CopyConfig(cfg) + + var testLog bytes.Buffer + testCfg.WarningHandler = rest.NewWarningWriter(&testLog, rest.WarningWriterOptions{}) + + cl, err := client.New(testCfg, client.Options{Cache: &client.CacheOptions{Reader: cache, DisableFor: []client.Object{&corev1.Namespace{}}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "wh-defined"}} + tns, err = clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(tns).NotTo(BeNil()) + defer deleteNamespace(ctx, tns) + + toCreate := &pkg.ChaosPod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example", + Namespace: tns.Name, + }, + // The ChaosPod CRD does not define Status, so the field is unknown to the API server, + // but field validation is not strict by default, so the API server returns a warning, + // and we need a warning to check whether suppression works. + Status: pkg.ChaosPodStatus{}, + } + err = cl.Create(ctx, toCreate) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + scannerTestLog := bufio.NewScanner(&testLog) + for scannerTestLog.Scan() { + line := scannerTestLog.Text() + if strings.Contains( + line, + "unknown field \"status\"", + ) { + return + } + } + defer Fail("expected to find one API server warning logged the config.WarningHandler") + + scanner := bufio.NewScanner(&log) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains( + line, + "unknown field \"status\"", + ) { + defer Fail("expected to find zero API server warnings in the client log") + break + } + } + }) + }) + Describe("New", func() { It("should return a new Client", func() { cl, err := client.New(cfg, client.Options{}) @@ -268,7 +336,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(cl.RESTMapper()).ToNot(BeNil()) }) - It("should use the provided reader cache if provided, on get and list", func() { + It("should use the provided reader cache if provided, on get and list", func(ctx SpecContext) { cache := &fakeReader{} cl, err := client.New(cfg, client.Options{Cache: &client.CacheOptions{Reader: cache}}) Expect(err).NotTo(HaveOccurred()) @@ -278,7 +346,17 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(cache.Called).To(Equal(2)) }) - It("should not use the provided reader cache if provided, on get and list for uncached GVKs", func() { + It("should propagate ErrResourceNotCached errors", func(ctx SpecContext) { + c := &fakeUncachedReader{} + cl, err := client.New(cfg, client.Options{Cache: &client.CacheOptions{Reader: c}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + Expect(errors.As(cl.Get(ctx, client.ObjectKey{Name: "test"}, &appsv1.Deployment{}), &errNotCached)).To(BeTrue()) + Expect(errors.As(cl.List(ctx, &appsv1.DeploymentList{}), &errNotCached)).To(BeTrue()) + Expect(c.Called).To(Equal(2)) + }) + + It("should not use the provided reader cache if provided, on get and list for uncached GVKs", func(ctx SpecContext) { cache := &fakeReader{} cl, err := client.New(cfg, client.Options{Cache: &client.CacheOptions{Reader: cache, DisableFor: []client.Object{&corev1.Namespace{}}}}) Expect(err).NotTo(HaveOccurred()) @@ -291,13 +369,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Describe("Create", func() { Context("with structured objects", func() { - It("should create a new object from a go struct", func() { + It("should create a new object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the object") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -308,13 +386,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(dep).To(Equal(actual)) }) - It("should create a new object non-namespace object from a go struct", func() { + It("should create a new object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the object") - err = cl.Create(context.TODO(), node) + err = cl.Create(ctx, node) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) @@ -325,7 +403,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(node).To(Equal(actual)) }) - It("should fail if the object already exists", func() { + It("should fail if the object already exists", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -333,7 +411,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC old := dep.DeepCopy() By("creating the object") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -341,24 +419,24 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual).NotTo(BeNil()) By("creating the object a second time") - err = cl.Create(context.TODO(), old) + err = cl.Create(ctx, old) Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) }) - It("should fail if the object does not pass server-side validation", func() { + It("should fail if the object does not pass server-side validation", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the pod, since required field Containers is empty") - err = cl.Create(context.TODO(), pod) + err = cl.Create(ctx, pod) Expect(err).To(HaveOccurred()) // TODO(seans): Add test to validate the returned error. Problems currently with // different returned error locally versus travis. }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -366,7 +444,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(cl).NotTo(BeNil()) By("creating the object fails") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -377,13 +455,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with the DryRun option", func() { - It("should not create a new object, global option", func() { - cl, err := client.New(cfg, client.Options{DryRun: pointer.Bool(true)}) + It("should not create a new object, global option", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{DryRun: ptr.To(true)}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the object (with DryRun)") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -392,13 +470,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual).To(Equal(&appsv1.Deployment{})) }) - It("should not create a new object, inline option", func() { + It("should not create a new object, inline option", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("creating the object (with DryRun)") - err = cl.Create(context.TODO(), dep, client.DryRunAll) + err = cl.Create(ctx, dep, client.DryRunAll) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -410,7 +488,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with unstructured objects", func() { - It("should create a new object from a go struct", func() { + It("should create a new object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -425,7 +503,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) By("creating the object") - err = cl.Create(context.TODO(), u) + err = cl.Create(ctx, u) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -433,7 +511,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual).NotTo(BeNil()) }) - It("should create a new non-namespace object ", func() { + It("should create a new non-namespace object ", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -448,7 +526,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) By("creating the object") - err = cl.Create(context.TODO(), node) + err = cl.Create(ctx, node) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) @@ -462,7 +540,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(u).To(Equal(au)) }) - It("should fail if the object already exists", func() { + It("should fail if the object already exists", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -470,7 +548,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC old := dep.DeepCopy() By("creating the object") - err = cl.Create(context.TODO(), dep) + err = cl.Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -486,12 +564,12 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) By("creating the object a second time") - err = cl.Create(context.TODO(), u) + err = cl.Create(ctx, u) Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) }) - It("should fail if the object does not pass server-side validation", func() { + It("should fail if the object does not pass server-side validation", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -504,7 +582,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "Pod", }) - err = cl.Create(context.TODO(), u) + err = cl.Create(ctx, u) Expect(err).To(HaveOccurred()) // TODO(seans): Add test to validate the returned error. Problems currently with // different returned error locally versus travis. @@ -513,17 +591,17 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with metadata objects", func() { - It("should fail with an error", func() { + It("should fail with an error", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) obj := metaOnlyFromObj(dep, scheme) - Expect(cl.Create(context.TODO(), obj)).NotTo(Succeed()) + Expect(cl.Create(ctx, obj)).NotTo(Succeed()) }) }) Context("with the DryRun option", func() { - It("should not create a new object from a go struct", func() { + It("should not create a new object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -538,7 +616,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) By("creating the object") - err = cl.Create(context.TODO(), u, client.DryRunAll) + err = cl.Create(ctx, u, client.DryRunAll) Expect(err).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -551,7 +629,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Describe("Update", func() { Context("with structured objects", func() { - It("should update an existing object from a go struct", func() { + It("should update an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -562,7 +640,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating the Deployment") dep.Annotations = map[string]string{"foo": "bar"} - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new annotation") @@ -572,7 +650,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should update and preserve type information", func() { + It("should update and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -583,14 +661,14 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating the Deployment") dep.SetGroupVersionKind(depGvk) - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) }) - It("should update an existing object non-namespace object from a go struct", func() { + It("should update an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -600,7 +678,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating the object") node.Annotations = map[string]string{"foo": "bar"} - err = cl.Update(context.TODO(), node) + err = cl.Update(ctx, node) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -610,13 +688,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("updating non-existent object") - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).To(HaveOccurred()) }) @@ -628,7 +706,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -641,7 +719,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating the Deployment") dep.Annotations = map[string]string{"foo": "bar"} - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -651,7 +729,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) }) Context("with unstructured objects", func() { - It("should update an existing object from a go struct", func() { + It("should update an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -669,7 +747,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", }) u.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.TODO(), u) + err = cl.Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new annotation") @@ -679,7 +757,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should update and preserve type information", func() { + It("should update and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -693,14 +771,14 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(scheme.Convert(dep, u, nil)).To(Succeed()) u.SetGroupVersionKind(depGvk) u.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.TODO(), u) + err = cl.Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(u.GroupVersionKind()).To(Equal(depGvk)) }) - It("should update an existing object non-namespace object from a go struct", func() { + It("should update an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -717,7 +795,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", }) u.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.TODO(), u) + err = cl.Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -726,7 +804,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual).NotTo(BeNil()) Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -735,25 +813,25 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC u := &unstructured.Unstructured{} Expect(scheme.Convert(dep, u, nil)).To(Succeed()) u.SetGroupVersionKind(depGvk) - err = cl.Update(context.TODO(), dep) + err = cl.Update(ctx, dep) Expect(err).To(HaveOccurred()) }) }) Context("with metadata objects", func() { - It("should fail with an error", func() { + It("should fail with an error", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) obj := metaOnlyFromObj(dep, scheme) - Expect(cl.Update(context.TODO(), obj)).NotTo(Succeed()) + Expect(cl.Update(ctx, obj)).NotTo(Succeed()) }) }) }) Describe("Patch", func() { Context("Metadata Client", func() { - It("should merge patch with options", func() { + It("should merge patch with options", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -769,7 +847,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC metadata.Labels["foo"] = "bar" testOption := &mockPatchOption{} - Expect(cl.Patch(context.TODO(), metadata, client.Merge, testOption)).To(Succeed()) + Expect(cl.Patch(ctx, metadata, client.Merge, testOption)).To(Succeed()) By("validating that patched metadata has new labels") actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -783,9 +861,150 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) }) + Describe("Apply", func() { + Context("Unstructured Client", func() { + It("should create and update a configMap using SSA", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + data := map[string]any{ + "some-key": "some-value", + } + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": "test-configmap", + "namespace": "default", + }, + "data": data, + }} + + err = cl.Apply(ctx, client.ApplyConfigurationFromUnstructured(obj), &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err := clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Get(ctx, obj.GetName(), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + actualData := map[string]any{} + for k, v := range cm.Data { + actualData[k] = v + } + + Expect(actualData).To(BeComparableTo(data)) + Expect(actualData).To(BeComparableTo(obj.Object["data"])) + + data = map[string]any{ + "a-new-key": "a-new-value", + } + obj.Object["data"] = data + unstructured.RemoveNestedField(obj.Object, "metadata", "managedFields") + + err = cl.Apply(ctx, client.ApplyConfigurationFromUnstructured(obj), &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err = clientset.CoreV1().ConfigMaps(obj.GetNamespace()).Get(ctx, obj.GetName(), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + actualData = map[string]any{} + for k, v := range cm.Data { + actualData[k] = v + } + + Expect(actualData).To(BeComparableTo(data)) + Expect(actualData).To(BeComparableTo(obj.Object["data"])) + }) + }) + + Context("Structured Client", func() { + It("should create and update a configMap using SSA", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + data := map[string]string{ + "some-key": "some-value", + } + obj := corev1applyconfigurations. + ConfigMap("test-configmap", "default"). + WithData(data) + + err = cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err := clientset.CoreV1().ConfigMaps(ptr.Deref(obj.GetNamespace(), "")).Get(ctx, ptr.Deref(obj.GetName(), ""), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(cm.Data).To(BeComparableTo(data)) + Expect(cm.Data).To(BeComparableTo(obj.Data)) + + data = map[string]string{ + "a-new-key": "a-new-value", + } + obj.Data = data + + err = cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + cm, err = clientset.CoreV1().ConfigMaps(ptr.Deref(obj.GetNamespace(), "")).Get(ctx, ptr.Deref(obj.GetName(), ""), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(cm.Data).To(BeComparableTo(data)) + Expect(cm.Data).To(BeComparableTo(obj.Data)) + }) + + It("should create a secret without SSA and later create update a secret using SSA", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + data := map[string][]byte{ + "some-key": []byte("some-value"), + } + secretObject := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-one", + Namespace: "default", + }, + Data: data, + } + + secretApplyConfiguration := corev1applyconfigurations. + Secret("secret-two", "default"). + WithData(data) + + err = cl.Create(ctx, secretObject) + Expect(err).NotTo(HaveOccurred()) + + err = cl.Apply(ctx, secretApplyConfiguration, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + secret, err := clientset.CoreV1().Secrets(ptr.Deref(secretApplyConfiguration.GetNamespace(), "")).Get(ctx, ptr.Deref(secretApplyConfiguration.GetName(), ""), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(secret.Data).To(BeComparableTo(data)) + Expect(secret.Data).To(BeComparableTo(secretApplyConfiguration.Data)) + + data = map[string][]byte{ + "some-key": []byte("some-new-value"), + } + secretApplyConfiguration.Data = data + + err = cl.Apply(ctx, secretApplyConfiguration, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).NotTo(HaveOccurred()) + + secret, err = clientset.CoreV1().Secrets(ptr.Deref(secretApplyConfiguration.GetNamespace(), "")).Get(ctx, ptr.Deref(secretApplyConfiguration.GetName(), ""), metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(secret.Data).To(BeComparableTo(data)) + Expect(secret.Data).To(BeComparableTo(secretApplyConfiguration.Data)) + }) + }) + }) + Describe("SubResourceClient", func() { Context("with structured objects", func() { - It("should be able to read the Scale subresource", func() { + It("should be able to read the Scale subresource", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -796,11 +1015,11 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("reading the scale subresource") scale := &autoscalingv1.Scale{} - err = cl.SubResource("scale").Get(ctx, dep, scale) + err = cl.SubResource("scale").Get(ctx, dep, scale, &client.SubResourceGetOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(scale.Spec.Replicas).To(Equal(*dep.Spec.Replicas)) }) - It("should be able to create ServiceAccount tokens", func() { + It("should be able to create ServiceAccount tokens", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -810,13 +1029,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect((err)).NotTo(HaveOccurred()) token := &authenticationv1.TokenRequest{} - err = cl.SubResource("token").Create(ctx, serviceAccount, token) + err = cl.SubResource("token").Create(ctx, serviceAccount, token, &client.SubResourceCreateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(token.Status.Token).NotTo(Equal("")) }) - It("should be able to create Pod evictions", func() { + It("should be able to create Pod evictions", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -830,9 +1049,9 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("Creating the eviction") eviction := &policyv1.Eviction{ - DeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: ptr(int64(0))}, + DeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))}, } - err = cl.SubResource("eviction").Create(ctx, pod, eviction) + err = cl.SubResource("eviction").Create(ctx, pod, eviction, &client.SubResourceCreateOptions{}) Expect((err)).NotTo(HaveOccurred()) By("Asserting the pod is gone") @@ -840,7 +1059,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should be able to create Pod bindings", func() { + It("should be able to create Pod bindings", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -856,7 +1075,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC binding := &corev1.Binding{ Target: corev1.ObjectReference{Name: node.Name}, } - err = cl.SubResource("binding").Create(ctx, pod, binding) + err = cl.SubResource("binding").Create(ctx, pod, binding, &client.SubResourceCreateOptions{}) Expect((err)).NotTo(HaveOccurred()) By("Asserting the pod is bound") @@ -865,7 +1084,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(pod.Spec.NodeName).To(Equal(node.Name)) }) - It("should be able to approve CSRs", func() { + It("should be able to approve CSRs", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -879,7 +1098,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Type: certificatesv1.CertificateApproved, Status: corev1.ConditionTrue, }) - err = cl.SubResource("approval").Update(ctx, csr) + err = cl.SubResource("approval").Update(ctx, csr, &client.SubResourceUpdateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Asserting the CSR is approved") @@ -889,7 +1108,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) }) - It("should be able to approve CSRs using Patch", func() { + It("should be able to approve CSRs using Patch", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -904,7 +1123,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Type: certificatesv1.CertificateApproved, Status: corev1.ConditionTrue, }) - err = cl.SubResource("approval").Patch(ctx, csr, patch) + err = cl.SubResource("approval").Patch(ctx, csr, patch, &client.SubResourcePatchOptions{}) Expect(err).NotTo(HaveOccurred()) By("Asserting the CSR is approved") @@ -914,7 +1133,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) }) - It("should be able to update the scale subresource", func() { + It("should be able to update the scale subresource", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -923,10 +1142,10 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - By("Updating the scale subresurce") + By("Updating the scale subresource") replicaCount := *dep.Spec.Replicas scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: replicaCount}} - err = cl.SubResource("scale").Update(ctx, dep, client.WithSubResourceBody(scale)) + err = cl.SubResource("scale").Update(ctx, dep, client.WithSubResourceBody(scale), &client.SubResourceUpdateOptions{}) Expect(err).NotTo(HaveOccurred()) By("Asserting replicas got updated") @@ -935,7 +1154,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) }) - It("should be able to patch the scale subresource", func() { + It("should be able to patch the scale subresource", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -948,7 +1167,35 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC replicaCount := *dep.Spec.Replicas patch := client.MergeFrom(&autoscalingv1.Scale{}) scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: replicaCount}} - err = cl.SubResource("scale").Patch(ctx, dep, patch, client.WithSubResourceBody(scale)) + err = cl.SubResource("scale").Patch(ctx, dep, patch, client.WithSubResourceBody(scale), &client.SubResourcePatchOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + + It("should be able to apply the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + replicaCount := *dep.Spec.Replicas + 1 + + By("Applying the scale subresurce") + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "foo") + Expect(err).NotTo(HaveOccurred()) + scale := autoscaling1applyconfigurations.Scale(). + WithSpec(autoscaling1applyconfigurations.ScaleSpec().WithReplicas(replicaCount)) + err = cl.SubResource("scale").Apply(ctx, deploymentAC, + &client.SubResourceApplyOptions{SubResourceBody: scale}, + client.FieldOwner("foo"), + client.ForceOwnership, + ) Expect(err).NotTo(HaveOccurred()) By("Asserting replicas got updated") @@ -959,8 +1206,8 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with unstructured objects", func() { - It("should be able to read the Scale subresource", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to read the Scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -984,8 +1231,8 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(found).To(BeTrue()) Expect(int32(val)).To(Equal(*dep.Spec.Replicas)) }) - It("should be able to create ServiceAccount tokens", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to create ServiceAccount tokens", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1012,8 +1259,8 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(val).NotTo(Equal("")) }) - It("should be able to create Pod evictions", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to create Pod evictions", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1045,8 +1292,8 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should be able to create Pod bindings", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to create Pod bindings", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1080,8 +1327,8 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(pod.Spec.NodeName).To(Equal(node.Name)) }) - It("should be able to approve CSRs", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to approve CSRs", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1111,8 +1358,8 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) }) - It("should be able to approve CSRs using Patch", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to approve CSRs using Patch", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1143,16 +1390,16 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) }) - It("should be able to update the scale subresource", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to update the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("Creating a deployment") dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - dep.APIVersion = "apps/v1" - dep.Kind = "Deployment" + dep.APIVersion = appsv1.SchemeGroupVersion.String() + dep.Kind = "Deployment" //nolint:goconst depUnstructured, err := toUnstructured(dep) Expect(err).NotTo(HaveOccurred()) @@ -1173,8 +1420,8 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) }) - It("should be able to patch the scale subresource", func() { - cl, err := client.New(cfg, client.Options{}) + It("should be able to patch the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1203,13 +1450,48 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).NotTo(HaveOccurred()) Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) }) + + It("should be able to apply the scale subresource", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{Scheme: runtime.NewScheme()}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = "apps/v1" + dep.Kind = "Deployment" + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + 1 + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + Expect(unstructured.SetNestedField(scale.Object, int64(replicaCount), "spec", "replicas")).NotTo(HaveOccurred()) + err = cl.SubResource("scale").Apply(ctx, + client.ApplyConfigurationFromUnstructured(depUnstructured), + &client.SubResourceApplyOptions{SubResourceBody: client.ApplyConfigurationFromUnstructured(scale)}, + client.FieldOwner("foo"), + client.ForceOwnership, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.GetAPIVersion()).To(Equal("autoscaling/v1")) + Expect(scale.GetKind()).To(Equal("Scale")) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) }) }) Describe("StatusClient", func() { Context("with structured objects", func() { - It("should update status of an existing object", func() { + It("should update status of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1220,7 +1502,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating the status of Deployment") dep.Status.Replicas = 1 - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status") @@ -1230,7 +1512,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) }) - It("should update status and preserve type information", func() { + It("should update status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1242,14 +1524,14 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating the status of Deployment") dep.SetGroupVersionKind(depGvk) dep.Status.Replicas = 1 - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) }) - It("should patch status and preserve type information", func() { + It("should patch status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1262,14 +1544,37 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC dep.SetGroupVersionKind(depGvk) depPatch := client.MergeFrom(dep.DeepCopy()) dep.Status.Replicas = 1 - err = cl.Status().Patch(context.TODO(), dep, depPatch) + err = cl.Status().Patch(ctx, dep, depPatch) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) }) - It("should not update spec of an existing object", func() { + It("should apply status", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("initially creating a Deployment") + dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Status.Replicas).To(BeEquivalentTo(0)) + + By("applying the status of Deployment") + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "foo") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(1)), + }) + Expect(cl.Status().Apply(ctx, deploymentAC, client.FieldOwner("foo"))).To(Succeed()) + + dep, err = clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Status.Replicas).To(BeEquivalentTo(1)) + }) + + It("should not update spec of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1282,7 +1587,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC var rc int32 = 1 dep.Status.Replicas = 1 dep.Spec.Replicas = &rc - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status and unchanged spec") @@ -1293,7 +1598,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(*actual.Spec.Replicas).To(BeEquivalentTo(replicaCount)) }) - It("should update an existing object non-namespace object", func() { + It("should update an existing object non-namespace object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1303,7 +1608,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating status of the object") node.Status.Phase = corev1.NodeRunning - err = cl.Status().Update(context.TODO(), node) + err = cl.Status().Update(ctx, node) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -1313,17 +1618,17 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Status.Phase).To(Equal(corev1.NodeRunning)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("updating status of a non-existent object") - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).To(HaveOccurred()) }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -1336,7 +1641,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating status of the Deployment") dep.Status.Replicas = 1 - err = cl.Status().Update(context.TODO(), dep) + err = cl.Status().Update(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -1351,7 +1656,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with unstructured objects", func() { - It("should update status of an existing object", func() { + It("should update status of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1364,7 +1669,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC u := &unstructured.Unstructured{} dep.Status.Replicas = 1 Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status") @@ -1374,7 +1679,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) }) - It("should update status and preserve type information", func() { + It("should update status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1387,14 +1692,14 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC u := &unstructured.Unstructured{} dep.Status.Replicas = 1 Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") Expect(u.GroupVersionKind()).To(Equal(depGvk)) }) - It("should patch status and preserve type information", func() { + It("should patch status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1408,7 +1713,35 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC depPatch := client.MergeFrom(dep.DeepCopy()) dep.Status.Replicas = 1 Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Patch(context.TODO(), u, depPatch) + err = cl.Status().Patch(ctx, u, depPatch) + Expect(err).NotTo(HaveOccurred()) + + By("validating updated Deployment has type information") + Expect(u.GroupVersionKind()).To(Equal(depGvk)) + + By("validating patched Deployment has new status") + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) + }) + + It("should apply status and preserve type information", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("initially creating a Deployment") + dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(dep.Status.Replicas).To(BeEquivalentTo(0)) + + By("applying the status of Deployment") + dep.Status.Replicas = 1 + dep.ManagedFields = nil // Must be unset in SSA requests + u := &unstructured.Unstructured{} + Expect(scheme.Convert(dep, u, nil)).To(Succeed()) + err = cl.Status().Apply(ctx, client.ApplyConfigurationFromUnstructured(u), client.FieldOwner("foo")) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") @@ -1421,7 +1754,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) }) - It("should not update spec of an existing object", func() { + It("should not update spec of an existing object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1436,7 +1769,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC dep.Status.Replicas = 1 dep.Spec.Replicas = &rc Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has new status and unchanged spec") @@ -1447,7 +1780,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(*actual.Spec.Replicas).To(BeEquivalentTo(replicaCount)) }) - It("should update an existing object non-namespace object", func() { + It("should update an existing object non-namespace object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1459,7 +1792,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC u := &unstructured.Unstructured{} node.Status.Phase = corev1.NodeRunning Expect(scheme.Convert(node, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validate updated Node had new annotation") @@ -1469,7 +1802,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(actual.Status.Phase).To(Equal(corev1.NodeRunning)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1477,7 +1810,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("updating status of a non-existent object") u := &unstructured.Unstructured{} Expect(scheme.Convert(dep, u, nil)).To(Succeed()) - err = cl.Status().Update(context.TODO(), u) + err = cl.Status().Update(ctx, u) Expect(err).To(HaveOccurred()) }) @@ -1492,15 +1825,15 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with metadata objects", func() { - It("should fail to update with an error", func() { + It("should fail to update with an error", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) obj := metaOnlyFromObj(dep, scheme) - Expect(cl.Status().Update(context.TODO(), obj)).NotTo(Succeed()) + Expect(cl.Status().Update(ctx, obj)).NotTo(Succeed()) }) - It("should patch status and preserve type information", func() { + It("should patch status and preserve type information", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1513,7 +1846,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC objPatch := client.MergeFrom(metaOnlyFromObj(dep, scheme)) dep.Annotations = map[string]string{"some-new-annotation": "some-new-value"} obj := metaOnlyFromObj(dep, scheme) - err = cl.Status().Patch(context.TODO(), obj, objPatch) + err = cl.Status().Patch(ctx, obj, objPatch) Expect(err).NotTo(HaveOccurred()) By("validating updated Deployment has type information") @@ -1530,7 +1863,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Describe("Delete", func() { Context("with structured objects", func() { - It("should delete an existing object from a go struct", func() { + It("should delete an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1541,7 +1874,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("deleting the Deployment") depName := dep.Name - err = cl.Delete(context.TODO(), dep) + err = cl.Delete(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1549,7 +1882,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).To(HaveOccurred()) }) - It("should delete an existing object non-namespace object from a go struct", func() { + It("should delete an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1560,7 +1893,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("deleting the Node") nodeName := node.Name - err = cl.Delete(context.TODO(), node) + err = cl.Delete(ctx, node) Expect(err).NotTo(HaveOccurred()) By("validating the Node no longer exists") @@ -1568,13 +1901,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).To(HaveOccurred()) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("Deleting node before it is ever created") - err = cl.Delete(context.TODO(), node) + err = cl.Delete(ctx, node) Expect(err).To(HaveOccurred()) }) @@ -1582,7 +1915,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -1594,7 +1927,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).NotTo(HaveOccurred()) By("deleting the Deployment fails") - err = cl.Delete(context.TODO(), dep) + err = cl.Delete(ctx, dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -1603,7 +1936,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) - It("should delete a collection of objects", func() { + It("should delete a collection of objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1622,7 +1955,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC dep2Name := dep2.Name By("deleting Deployments") - err = cl.DeleteAllOf(context.TODO(), dep, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) + err = cl.DeleteAllOf(ctx, dep, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1633,7 +1966,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) }) Context("with unstructured objects", func() { - It("should delete an existing object from a go struct", func() { + It("should delete an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1651,7 +1984,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "Deployment", Version: "v1", }) - err = cl.Delete(context.TODO(), u) + err = cl.Delete(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1659,7 +1992,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).To(HaveOccurred()) }) - It("should delete an existing object non-namespace object from a go struct", func() { + It("should delete an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1677,7 +2010,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "Node", Version: "v1", }) - err = cl.Delete(context.TODO(), u) + err = cl.Delete(ctx, u) Expect(err).NotTo(HaveOccurred()) By("validating the Node no longer exists") @@ -1685,7 +2018,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).To(HaveOccurred()) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1698,11 +2031,11 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "Node", Version: "v1", }) - err = cl.Delete(context.TODO(), node) + err = cl.Delete(ctx, node) Expect(err).To(HaveOccurred()) }) - It("should delete a collection of object", func() { + It("should delete a collection of object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1728,7 +2061,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "Deployment", Version: "v1", }) - err = cl.DeleteAllOf(context.TODO(), u, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) + err = cl.DeleteAllOf(ctx, u, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1739,7 +2072,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) }) Context("with metadata objects", func() { - It("should delete an existing object from a go struct", func() { + It("should delete an existing object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1750,7 +2083,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("deleting the Deployment") metaObj := metaOnlyFromObj(dep, scheme) - err = cl.Delete(context.TODO(), metaObj) + err = cl.Delete(ctx, metaObj) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1758,7 +2091,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).To(HaveOccurred()) }) - It("should delete an existing object non-namespace object from a go struct", func() { + It("should delete an existing object non-namespace object from a go struct", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1769,7 +2102,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("deleting the Node") metaObj := metaOnlyFromObj(node, scheme) - err = cl.Delete(context.TODO(), metaObj) + err = cl.Delete(ctx, metaObj) Expect(err).NotTo(HaveOccurred()) By("validating the Node no longer exists") @@ -1777,18 +2110,18 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(err).To(HaveOccurred()) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) By("Deleting node before it is ever created") metaObj := metaOnlyFromObj(node, scheme) - err = cl.Delete(context.TODO(), metaObj) + err = cl.Delete(ctx, metaObj) Expect(err).To(HaveOccurred()) }) - It("should delete a collection of object", func() { + It("should delete a collection of object", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1808,7 +2141,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("deleting Deployments") metaObj := metaOnlyFromObj(dep, scheme) - err = cl.DeleteAllOf(context.TODO(), metaObj, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) + err = cl.DeleteAllOf(ctx, metaObj, client.InNamespace(ns), client.MatchingLabels(dep.ObjectMeta.Labels)) Expect(err).NotTo(HaveOccurred()) By("validating the Deployment no longer exists") @@ -1822,7 +2155,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Describe("Get", func() { Context("with structured objects", func() { - It("should fetch an existing object for a go struct", func() { + It("should fetch an existing object for a go struct", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1834,7 +2167,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("fetching the created Deployment") var actual appsv1.Deployment key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) @@ -1842,7 +2175,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(dep).To(Equal(&actual)) }) - It("should fetch an existing non-namespace object for a go struct", func() { + It("should fetch an existing non-namespace object for a go struct", func(ctx SpecContext) { By("first creating the object") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1854,14 +2187,14 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("retrieving node through client") var actual corev1.Node key := client.ObjectKey{Namespace: ns, Name: node.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(node).To(Equal(&actual)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1869,7 +2202,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("fetching object that has not been created yet") key := client.ObjectKey{Namespace: ns, Name: dep.Name} var actual appsv1.Deployment - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).To(HaveOccurred()) }) @@ -1877,7 +2210,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) - It("should fail if the object cannot be mapped to a GVK", func() { + It("should fail if the object cannot be mapped to a GVK", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1891,7 +2224,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("fetching the created Deployment fails") var actual appsv1.Deployment key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) }) @@ -1903,8 +2236,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC // Test this with an integrated type and a CRD to make sure it covers both proto // and json deserialization. for idx, object := range []client.Object{&corev1.ConfigMap{}, &pkg.ChaosPod{}} { - idx, object := idx, object - It(fmt.Sprintf("should not retain any data in the obj variable that is not on the server for %T", object), func() { + It(fmt.Sprintf("should not retain any data in the obj variable that is not on the server for %T", object), func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1929,7 +2261,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with unstructured objects", func() { - It("should fetch an existing object", func() { + It("should fetch an existing object", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1950,15 +2282,16 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", }) key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) By("validating the fetched Deployment equals the created one") - Expect(u).To(Equal(&actual)) + unstructured.RemoveNestedField(actual.Object, "spec", "template", "metadata", "creationTimestamp") + Expect(u).To(BeComparableTo(&actual)) }) - It("should fetch an existing non-namespace object", func() { + It("should fetch an existing non-namespace object", func(ctx SpecContext) { By("first creating the Node") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1979,7 +2312,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", }) key := client.ObjectKey{Namespace: ns, Name: node.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) @@ -1987,7 +2320,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(u).To(Equal(&actual)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1995,11 +2328,11 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("fetching object that has not been created yet") key := client.ObjectKey{Namespace: ns, Name: dep.Name} u := &unstructured.Unstructured{} - err = cl.Get(context.TODO(), key, u) + err = cl.Get(ctx, key, u) Expect(err).To(HaveOccurred()) }) - It("should not retain any data in the obj variable that is not on the server", func() { + It("should not retain any data in the obj variable that is not on the server", func(ctx SpecContext) { object := &unstructured.Unstructured{} cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -2024,7 +2357,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) }) Context("with metadata objects", func() { - It("should fetch an existing object for a go struct", func() { + It("should fetch an existing object for a go struct", func(ctx SpecContext) { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2042,7 +2375,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC } actual.SetGroupVersionKind(gvk) key := client.ObjectKey{Namespace: ns, Name: dep.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) @@ -2053,7 +2386,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(metaOnlyFromObj(dep, scheme)).To(Equal(&actual)) }) - It("should fetch an existing non-namespace object for a go struct", func() { + It("should fetch an existing non-namespace object for a go struct", func(ctx SpecContext) { By("first creating the object") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2069,14 +2402,14 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "Node", }) key := client.ObjectKey{Namespace: ns, Name: node.Name} - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(metaOnlyFromObj(node, scheme)).To(Equal(&actual)) }) - It("should fail if the object does not exist", func() { + It("should fail if the object does not exist", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -2089,7 +2422,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "Deployment", }) - err = cl.Get(context.TODO(), key, &actual) + err = cl.Get(ctx, key, &actual) Expect(err).To(HaveOccurred()) }) @@ -2101,7 +2434,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) - It("should not retain any data in the obj variable that is not on the server", func() { + It("should not retain any data in the obj variable that is not on the server", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -2126,7 +2459,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Describe("List", func() { Context("with structured objects", func() { - It("should fetch collection of objects", func() { + It("should fetch collection of objects", func(ctx SpecContext) { By("creating an initial object") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2136,7 +2469,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing all objects of that type in the cluster") deps := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), deps)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, deps)).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) hasDep := false @@ -2149,7 +2482,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(hasDep).To(BeTrue()) }) - It("should fetch unstructured collection of objects", func() { + It("should fetch unstructured collection of objects", func(ctx SpecContext) { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2164,7 +2497,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps) + err = cl.List(ctx, deps) Expect(err).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) @@ -2183,7 +2516,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(hasDep).To(BeTrue()) }) - It("should fetch unstructured collection of objects, even if scheme is empty", func() { + It("should fetch unstructured collection of objects, even if scheme is empty", func(ctx SpecContext) { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2198,7 +2531,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps) + err = cl.List(ctx, deps) Expect(err).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) @@ -2212,20 +2545,20 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(hasDep).To(BeTrue()) }) - It("should return an empty list if there are no matching objects", func() { + It("should return an empty list if there are no matching objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) By("listing all Deployments in the cluster") deps := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), deps)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, deps)).NotTo(HaveOccurred()) By("validating no Deployments are returned") Expect(deps.Items).To(BeEmpty()) }) // TODO(seans): get label selector test working - It("should filter results by label selector", func() { + It("should filter results by label selector", func(ctx SpecContext) { By("creating a Deployment with the app=frontend label") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -2272,7 +2605,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing all Deployments with label app=backend") deps := &appsv1.DeploymentList{} labels := map[string]string{"app": "backend"} - err = cl.List(context.Background(), deps, client.MatchingLabels(labels)) + err = cl.List(ctx, deps, client.MatchingLabels(labels)) Expect(err).NotTo(HaveOccurred()) By("only the Deployment with the backend label is returned") @@ -2285,7 +2618,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteDeployment(ctx, depBackend, ns) }) - It("should filter results by namespace selector", func() { + It("should filter results by namespace selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-1") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -2329,7 +2662,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing all Deployments in test-namespace-1") deps := &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, client.InNamespace("test-namespace-1")) + err = cl.List(ctx, deps, client.InNamespace("test-namespace-1")) Expect(err).NotTo(HaveOccurred()) By("only the Deployment in test-namespace-1 is returned") @@ -2344,7 +2677,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteNamespace(ctx, tns2) }) - It("should filter results by field selector", func() { + It("should filter results by field selector", func(ctx SpecContext) { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -2382,7 +2715,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing all Deployments with field metadata.name=deployment-backend") deps := &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.MatchingFields{"metadata.name": "deployment-backend"}) Expect(err).NotTo(HaveOccurred()) @@ -2396,7 +2729,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteDeployment(ctx, depBackend, ns) }) - It("should filter results by namespace selector and label selector", func() { + It("should filter results by namespace selector and label selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-3 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-3"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -2469,7 +2802,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing all Deployments in test-namespace-3 with label app=frontend") deps := &appsv1.DeploymentList{} labels := map[string]string{"app": "frontend"} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.InNamespace("test-namespace-3"), client.MatchingLabels(labels), ) @@ -2489,7 +2822,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteNamespace(ctx, tns4) }) - It("should filter results using limit and continue options", func() { + It("should filter results using limit and continue options", func(ctx SpecContext) { makeDeployment := func(suffix string) *appsv1.Deployment { return &appsv1.Deployment{ @@ -2534,7 +2867,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing 1 deployment when limit=1 is used") deps := &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.Limit(1), ) Expect(err).NotTo(HaveOccurred()) @@ -2547,7 +2880,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing the next deployment when previous continuation token is used and limit=1") deps = &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.Limit(1), client.Continue(continueToken), ) @@ -2561,7 +2894,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC By("listing the 2 remaining deployments when previous continuation token is used without a limit") deps = &appsv1.DeploymentList{} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.Continue(continueToken), ) Expect(err).NotTo(HaveOccurred()) @@ -2586,7 +2919,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) Context("with unstructured objects", func() { - It("should fetch collection of objects", func() { + It("should fetch collection of objects", func(ctx SpecContext) { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2601,7 +2934,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps) + err = cl.List(ctx, deps) Expect(err).NotTo(HaveOccurred()) Expect(deps.Items).NotTo(BeEmpty()) @@ -2615,7 +2948,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(hasDep).To(BeTrue()) }) - It("should return an empty list if there are no matching objects", func() { + It("should return an empty list if there are no matching objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -2626,13 +2959,13 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", Version: "v1", }) - Expect(cl.List(context.Background(), deps)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, deps)).NotTo(HaveOccurred()) By("validating no Deployments are returned") Expect(deps.Items).To(BeEmpty()) }) - It("should filter results by namespace selector", func() { + It("should filter results by namespace selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-5") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-5"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -2681,7 +3014,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps, client.InNamespace("test-namespace-5")) + err = cl.List(ctx, deps, client.InNamespace("test-namespace-5")) Expect(err).NotTo(HaveOccurred()) By("only the Deployment in test-namespace-5 is returned") @@ -2696,7 +3029,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteNamespace(ctx, tns2) }) - It("should filter results by field selector", func() { + It("should filter results by field selector", func(ctx SpecContext) { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -2739,7 +3072,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", Version: "v1", }) - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.MatchingFields{"metadata.name": "deployment-backend"}) Expect(err).NotTo(HaveOccurred()) @@ -2753,7 +3086,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteDeployment(ctx, depBackend, ns) }) - It("should filter results by namespace selector and label selector", func() { + It("should filter results by namespace selector and label selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-7 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-7"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -2831,7 +3164,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", }) labels := map[string]string{"app": "frontend"} - err = cl.List(context.Background(), deps, + err = cl.List(ctx, deps, client.InNamespace("test-namespace-7"), client.MatchingLabels(labels)) Expect(err).NotTo(HaveOccurred()) @@ -2858,7 +3191,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC }) }) Context("with metadata objects", func() { - It("should fetch collection of objects", func() { + It("should fetch collection of objects", func(ctx SpecContext) { By("creating an initial object") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2874,7 +3207,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC } metaList := &metav1.PartialObjectMetadataList{} metaList.SetGroupVersionKind(gvk) - Expect(cl.List(context.Background(), metaList)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, metaList)).NotTo(HaveOccurred()) By("validating that the list GVK has been preserved") Expect(metaList.GroupVersionKind()).To(Equal(gvk)) @@ -2897,7 +3230,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Expect(hasDep).To(BeTrue()) }) - It("should return an empty list if there are no matching objects", func() { + It("should return an empty list if there are no matching objects", func(ctx SpecContext) { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -2908,14 +3241,14 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "DeploymentList", }) - Expect(cl.List(context.Background(), metaList)).NotTo(HaveOccurred()) + Expect(cl.List(ctx, metaList)).NotTo(HaveOccurred()) By("validating no Deployments are returned") Expect(metaList.Items).To(BeEmpty()) }) // TODO(seans): get label selector test working - It("should filter results by label selector", func() { + It("should filter results by label selector", func(ctx SpecContext) { By("creating a Deployment with the app=frontend label") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -2967,7 +3300,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", }) labels := map[string]string{"app": "backend"} - err = cl.List(context.Background(), metaList, client.MatchingLabels(labels)) + err = cl.List(ctx, metaList, client.MatchingLabels(labels)) Expect(err).NotTo(HaveOccurred()) By("only the Deployment with the backend label is returned") @@ -2980,7 +3313,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteDeployment(ctx, depBackend, ns) }) - It("should filter results by namespace selector", func() { + It("should filter results by namespace selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-1") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -3029,7 +3362,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, client.InNamespace("test-namespace-1")) + err = cl.List(ctx, metaList, client.InNamespace("test-namespace-1")) Expect(err).NotTo(HaveOccurred()) By("only the Deployment in test-namespace-1 is returned") @@ -3044,7 +3377,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteNamespace(ctx, tns2) }) - It("should filter results by field selector", func() { + It("should filter results by field selector", func(ctx SpecContext) { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -3087,7 +3420,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.MatchingFields{"metadata.name": "deployment-backend"}) Expect(err).NotTo(HaveOccurred()) @@ -3101,7 +3434,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteDeployment(ctx, depBackend, ns) }) - It("should filter results by namespace selector and label selector", func() { + It("should filter results by namespace selector and label selector", func(ctx SpecContext) { By("creating a Deployment in test-namespace-3 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-3"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -3179,7 +3512,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Kind: "DeploymentList", }) labels := map[string]string{"app": "frontend"} - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.InNamespace("test-namespace-3"), client.MatchingLabels(labels), ) @@ -3199,8 +3532,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC deleteNamespace(ctx, tns4) }) - It("should filter results using limit and continue options", func() { - + It("should filter results using limit and continue options", func(ctx SpecContext) { makeDeployment := func(suffix string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -3249,7 +3581,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.Limit(1), ) Expect(err).NotTo(HaveOccurred()) @@ -3267,7 +3599,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.Limit(1), client.Continue(continueToken), ) @@ -3286,7 +3618,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC Version: "v1", Kind: "DeploymentList", }) - err = cl.List(context.Background(), metaList, + err = cl.List(ctx, metaList, client.Continue(continueToken), ) Expect(err).NotTo(HaveOccurred()) @@ -3585,7 +3917,7 @@ U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC var _ = Describe("ClientWithCache", func() { Describe("Get", func() { - It("should call cache reader when structured object", func() { + It("should call cache reader when structured object", func(ctx SpecContext) { cachedReader := &fakeReader{} cl, err := client.New(cfg, client.Options{ Cache: &client.CacheOptions{ @@ -3595,14 +3927,14 @@ var _ = Describe("ClientWithCache", func() { Expect(err).NotTo(HaveOccurred()) var actual appsv1.Deployment key := client.ObjectKey{Namespace: "ns", Name: "name"} - Expect(cl.Get(context.TODO(), key, &actual)).To(Succeed()) + Expect(cl.Get(ctx, key, &actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) When("getting unstructured objects", func() { var dep *appsv1.Deployment - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "deployment1", @@ -3619,17 +3951,17 @@ var _ = Describe("ClientWithCache", func() { }, } var err error - dep, err = clientset.AppsV1().Deployments("default").Create(context.Background(), dep, metav1.CreateOptions{}) + dep, err = clientset.AppsV1().Deployments("default").Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { Expect(clientset.AppsV1().Deployments("default").Delete( - context.Background(), + ctx, dep.Name, metav1.DeleteOptions{}, )).To(Succeed()) }) - It("should call client reader when not cached", func() { + It("should call client reader when not cached", func(ctx SpecContext) { cachedReader := &fakeReader{} cl, err := client.New(cfg, client.Options{ Cache: &client.CacheOptions{ @@ -3646,10 +3978,10 @@ var _ = Describe("ClientWithCache", func() { }) actual.SetName(dep.Name) key := client.ObjectKey{Namespace: dep.Namespace, Name: dep.Name} - Expect(cl.Get(context.TODO(), key, actual)).To(Succeed()) + Expect(cl.Get(ctx, key, actual)).To(Succeed()) Expect(0).To(Equal(cachedReader.Called)) }) - It("should call cache reader when cached", func() { + It("should call cache reader when cached", func(ctx SpecContext) { cachedReader := &fakeReader{} cl, err := client.New(cfg, client.Options{ Cache: &client.CacheOptions{ @@ -3667,13 +3999,13 @@ var _ = Describe("ClientWithCache", func() { }) actual.SetName(dep.Name) key := client.ObjectKey{Namespace: dep.Namespace, Name: dep.Name} - Expect(cl.Get(context.TODO(), key, actual)).To(Succeed()) + Expect(cl.Get(ctx, key, actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) }) }) Describe("List", func() { - It("should call cache reader when structured object", func() { + It("should call cache reader when structured object", func(ctx SpecContext) { cachedReader := &fakeReader{} cl, err := client.New(cfg, client.Options{ Cache: &client.CacheOptions{ @@ -3682,12 +4014,12 @@ var _ = Describe("ClientWithCache", func() { }) Expect(err).NotTo(HaveOccurred()) var actual appsv1.DeploymentList - Expect(cl.List(context.Background(), &actual)).To(Succeed()) + Expect(cl.List(ctx, &actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) When("listing unstructured objects", func() { - It("should call client reader when not cached", func() { + It("should call client reader when not cached", func(ctx SpecContext) { cachedReader := &fakeReader{} cl, err := client.New(cfg, client.Options{ Cache: &client.CacheOptions{ @@ -3702,10 +4034,10 @@ var _ = Describe("ClientWithCache", func() { Kind: "DeploymentList", Version: "v1", }) - Expect(cl.List(context.Background(), actual)).To(Succeed()) + Expect(cl.List(ctx, actual)).To(Succeed()) Expect(0).To(Equal(cachedReader.Called)) }) - It("should call cache reader when cached", func() { + It("should call cache reader when cached", func(ctx SpecContext) { cachedReader := &fakeReader{} cl, err := client.New(cfg, client.Options{ Cache: &client.CacheOptions{ @@ -3721,7 +4053,7 @@ var _ = Describe("ClientWithCache", func() { Kind: "DeploymentList", Version: "v1", }) - Expect(cl.List(context.Background(), actual)).To(Succeed()) + Expect(cl.List(ctx, actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) }) @@ -3938,8 +4270,18 @@ func (f *fakeReader) List(ctx context.Context, list client.ObjectList, opts ...c return nil } -func ptr[T any](to T) *T { - return &to +type fakeUncachedReader struct { + Called int +} + +func (f *fakeUncachedReader) Get(_ context.Context, _ client.ObjectKey, _ client.Object, opts ...client.GetOption) error { + f.Called++ + return &cache.ErrResourceNotCached{} +} + +func (f *fakeUncachedReader) List(_ context.Context, _ client.ObjectList, _ ...client.ListOption) error { + f.Called++ + return &cache.ErrResourceNotCached{} } func toUnstructured(o client.Object) (*unstructured.Unstructured, error) { diff --git a/pkg/client/config/config.go b/pkg/client/config/config.go index 5f0a6d4b1d..1c39f4d854 100644 --- a/pkg/client/config/config.go +++ b/pkg/client/config/config.go @@ -61,8 +61,8 @@ func RegisterFlags(fs *flag.FlagSet) { // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // -// It also applies saner defaults for QPS and burst based on the Kubernetes -// controller manager defaults (20 QPS, 30 burst) +// The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and +// fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. // // Config precedence: // @@ -81,8 +81,8 @@ func GetConfig() (*rest.Config, error) { // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // -// It also applies saner defaults for QPS and burst based on the Kubernetes -// controller manager defaults (20 QPS, 30 burst) +// The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and +// fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. // // Config precedence: // @@ -99,10 +99,9 @@ func GetConfigWithContext(context string) (*rest.Config, error) { return nil, err } if cfg.QPS == 0.0 { - cfg.QPS = 20.0 - } - if cfg.Burst == 0 { - cfg.Burst = 30 + // Disable client-side ratelimer by default, we can rely on + // API priority and fairness + cfg.QPS = -1 } return cfg, nil } @@ -170,6 +169,9 @@ func loadConfigWithContext(apiServerURL string, loader clientcmd.ClientConfigLoa // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. // +// The returned `*rest.Config` has client-side ratelimting disabled as we can rely on API priority and +// fairness. Set its QPS to a value equal or bigger than 0 to re-enable it. +// // Will log an error and exit if there is an error creating the rest.Config. func GetConfigOrDie() *rest.Config { config, err := GetConfig() diff --git a/pkg/client/config/config_test.go b/pkg/client/config/config_test.go index 058ff33c1f..bbaeb2e2bd 100644 --- a/pkg/client/config/config_test.go +++ b/pkg/client/config/config_test.go @@ -52,7 +52,7 @@ var _ = Describe("Config", func() { }) AfterEach(func() { - os.Unsetenv(clientcmd.RecommendedConfigPathEnvVar) + _ = os.Unsetenv(clientcmd.RecommendedConfigPathEnvVar) kubeconfig = "" clientcmd.RecommendedHomeFile = origRecommendedHomeFile @@ -72,6 +72,7 @@ var _ = Describe("Config", func() { cfg, err := GetConfigWithContext(tc.context) Expect(err).NotTo(HaveOccurred()) Expect(cfg.Host).To(Equal(tc.wantHost)) + Expect(cfg.QPS).To(Equal(float32(-1))) }) } } @@ -82,8 +83,8 @@ var _ = Describe("Config", func() { Expect(err).NotTo(HaveOccurred()) cfg, err := GetConfigWithContext("") - Expect(cfg).To(BeNil()) Expect(err).To(HaveOccurred()) + Expect(cfg).To(BeNil()) }) }) @@ -191,7 +192,7 @@ func setConfigs(tc testCase, dir string) { func createFiles(files map[string]string, dir string) error { for path, data := range files { - if err := os.WriteFile(filepath.Join(dir, path), []byte(data), 0644); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(dir, path), []byte(data), 0644); err != nil { return err } } diff --git a/pkg/client/dryrun.go b/pkg/client/dryrun.go index bbcdd38321..fb7012200f 100644 --- a/pkg/client/dryrun.go +++ b/pkg/client/dryrun.go @@ -82,6 +82,10 @@ func (c *dryRunClient) Patch(ctx context.Context, obj Object, patch Patch, opts return c.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...) } +func (c *dryRunClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + return c.client.Apply(ctx, obj, append(opts, DryRunAll)...) +} + // Get implements client.Client. func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { return c.client.Get(ctx, key, obj, opts...) @@ -128,3 +132,7 @@ func (sw *dryRunSubResourceClient) Update(ctx context.Context, obj Object, opts func (sw *dryRunSubResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { return sw.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...) } + +func (sw *dryRunSubResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + return sw.client.Apply(ctx, obj, append(opts, DryRunAll)...) +} diff --git a/pkg/client/dryrun_test.go b/pkg/client/dryrun_test.go index 72907fefab..35a9b63869 100644 --- a/pkg/client/dryrun_test.go +++ b/pkg/client/dryrun_test.go @@ -17,7 +17,6 @@ limitations under the License. package client_test import ( - "context" "fmt" "sync/atomic" @@ -28,7 +27,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" + appsv1applyconfigurations "k8s.io/client-go/applyconfigurations/apps/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -38,16 +38,15 @@ var _ = Describe("DryRunClient", func() { var count uint64 = 0 var replicaCount int32 = 2 var ns = "default" - ctx := context.Background() getClient := func() client.Client { - cl, err := client.New(cfg, client.Options{DryRun: pointer.Bool(true)}) + cl, err := client.New(cfg, client.Options{DryRun: ptr.To(true)}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) return cl } - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { atomic.AddUint64(&count, 1) dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -72,11 +71,11 @@ var _ = Describe("DryRunClient", func() { Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully Get an object", func() { + It("should successfully Get an object", func(ctx SpecContext) { name := types.NamespacedName{Namespace: ns, Name: dep.Name} result := &appsv1.Deployment{} @@ -84,7 +83,7 @@ var _ = Describe("DryRunClient", func() { Expect(result).To(BeEquivalentTo(dep)) }) - It("should successfully List objects", func() { + It("should successfully List objects", func(ctx SpecContext) { result := &appsv1.DeploymentList{} opts := client.MatchingLabels(dep.Labels) @@ -94,7 +93,7 @@ var _ = Describe("DryRunClient", func() { Expect(result.Items[0]).To(BeEquivalentTo(*dep)) }) - It("should not create an object", func() { + It("should not create an object", func(ctx SpecContext) { newDep := dep.DeepCopy() newDep.Name = "new-deployment" @@ -104,7 +103,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should not create an object with opts", func() { + It("should not create an object with opts", func(ctx SpecContext) { newDep := dep.DeepCopy() newDep.Name = "new-deployment" opts := &client.CreateOptions{DryRun: []string{"Bye", "Pippa"}} @@ -115,7 +114,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should refuse a create request for an invalid object", func() { + It("should refuse a create request for an invalid object", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Spec.Template.Spec.Containers = nil @@ -123,7 +122,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsInvalid(err)).To(BeTrue()) }) - It("should not change objects via update", func() { + It("should not change objects via update", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 @@ -135,7 +134,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via update with opts", func() { + It("should not change objects via update with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 opts := &client.UpdateOptions{DryRun: []string{"Bye", "Pippa"}} @@ -148,7 +147,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should refuse an update request for an invalid change", func() { + It("should refuse an update request for an invalid change", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Spec.Template.Spec.Containers = nil @@ -156,7 +155,7 @@ var _ = Describe("DryRunClient", func() { Expect(apierrors.IsInvalid(err)).To(BeTrue()) }) - It("should not change objects via patch", func() { + It("should not change objects via patch", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 @@ -168,7 +167,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via patch with opts", func() { + It("should not change objects via patch with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() *changedDep.Spec.Replicas = 2 opts := &client.PatchOptions{DryRun: []string{"Bye", "Pippa"}} @@ -181,7 +180,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not delete objects", func() { + It("should not delete objects", func(ctx SpecContext) { Expect(getClient().Delete(ctx, dep)).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) @@ -190,7 +189,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not delete objects with opts", func() { + It("should not delete objects with opts", func(ctx SpecContext) { opts := &client.DeleteOptions{DryRun: []string{"Bye", "Pippa"}} Expect(getClient().Delete(ctx, dep, opts)).NotTo(HaveOccurred()) @@ -201,7 +200,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not delete objects via deleteAllOf", func() { + It("should not delete objects via deleteAllOf", func(ctx SpecContext) { opts := []client.DeleteAllOfOption{client.InNamespace(ns), client.MatchingLabels(dep.Labels)} Expect(getClient().DeleteAllOf(ctx, dep, opts...)).NotTo(HaveOccurred()) @@ -212,7 +211,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via update status", func() { + It("should not change objects via update status", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 @@ -224,7 +223,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via update status with opts", func() { + It("should not change objects via update status with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 opts := &client.SubResourceUpdateOptions{UpdateOptions: client.UpdateOptions{DryRun: []string{"Bye", "Pippa"}}} @@ -237,7 +236,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via status patch", func() { + It("should not change objects via status patch", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 @@ -249,7 +248,7 @@ var _ = Describe("DryRunClient", func() { Expect(actual).To(BeEquivalentTo(dep)) }) - It("should not change objects via status patch with opts", func() { + It("should not change objects via status patch with opts", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 @@ -262,4 +261,36 @@ var _ = Describe("DryRunClient", func() { Expect(actual).NotTo(BeNil()) Expect(actual).To(BeEquivalentTo(dep)) }) + + It("should not change objects via status apply", func(ctx SpecContext) { + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "test-owner") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(99)), + }) + + Expect(getClient().Status().Apply(ctx, deploymentAC, client.FieldOwner("test-owner"))).NotTo(HaveOccurred()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual).To(BeEquivalentTo(dep)) + }) + + It("should not change objects via status apply with opts", func(ctx SpecContext) { + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "test-owner") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(99)), + }) + + opts := &client.SubResourceApplyOptions{ApplyOptions: client.ApplyOptions{DryRun: []string{"Bye", "Pippa"}}} + + Expect(getClient().Status().Apply(ctx, deploymentAC, client.FieldOwner("test-owner"), opts)).NotTo(HaveOccurred()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual).To(BeEquivalentTo(dep)) + }) }) diff --git a/pkg/client/example_test.go b/pkg/client/example_test.go index 7d4cb8c616..390dc10143 100644 --- a/pkg/client/example_test.go +++ b/pkg/client/example_test.go @@ -25,11 +25,15 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) var ( @@ -53,6 +57,26 @@ func ExampleNew() { } } +func ExampleNew_suppress_warnings() { + cfg := config.GetConfigOrDie() + // Use a rest.WarningHandlerWithContext that discards warning messages. + cfg.WarningHandlerWithContext = rest.NoWarnings{} + + cl, err := client.New(cfg, client.Options{}) + if err != nil { + fmt.Println("failed to create client") + os.Exit(1) + } + + podList := &corev1.PodList{} + + err = cl.List(context.Background(), podList, client.InNamespace("default")) + if err != nil { + fmt.Printf("failed to list pods in namespace default: %v\n", err) + os.Exit(1) + } +} + // This example shows how to use the client with typed and unstructured objects to retrieve an object. func ExampleClient_get() { // Using a typed object. @@ -159,7 +183,7 @@ func ExampleClient_update() { Namespace: "namespace", Name: "name", }, pod) - pod.SetFinalizers(append(pod.GetFinalizers(), "new-finalizer")) + controllerutil.AddFinalizer(pod, "new-finalizer") _ = c.Update(context.Background(), pod) // Using a unstructured object. @@ -173,7 +197,7 @@ func ExampleClient_update() { Namespace: "namespace", Name: "name", }, u) - u.SetFinalizers(append(u.GetFinalizers(), "new-finalizer")) + controllerutil.AddFinalizer(u, "new-finalizer") _ = c.Update(context.Background(), u) } @@ -188,6 +212,18 @@ func ExampleClient_patch() { }, client.RawPatch(types.StrategicMergePatchType, patch)) } +// This example shows how to use the client with unstructured objects to create/patch objects using Server Side Apply, +// "k8s.io/apimachinery/pkg/runtime".DefaultUnstructuredConverter.ToUnstructured is used to convert an object into map[string]any representation, +// which is then set as an "Object" field in *unstructured.Unstructured struct, which implements client.Object. +func ExampleClient_apply() { + // Using a typed object. + configMap := corev1ac.ConfigMap("name", "namespace").WithData(map[string]string{"key": "value"}) + // c is a created client. + u := &unstructured.Unstructured{} + u.Object, _ = runtime.DefaultUnstructuredConverter.ToUnstructured(configMap) + _ = c.Patch(context.Background(), u, client.Apply, client.ForceOwnership, client.FieldOwner("field-owner")) +} + // This example shows how to use the client with typed and unstructured objects to patch objects' status. func ExampleClient_patchStatus() { u := &unstructured.Unstructured{} @@ -247,7 +283,7 @@ func ExampleClient_deleteAllOf() { } // This example shows how to set up and consume a field selector over a pod's volumes' secretName field. -func ExampleFieldIndexer_secretName() { +func ExampleFieldIndexer_secretNameNode() { // someIndexer is a FieldIndexer over a Cache _ = someIndexer.IndexField(context.TODO(), &corev1.Pod{}, "spec.volumes.secret.secretName", func(o client.Object) []string { var res []string @@ -261,8 +297,20 @@ func ExampleFieldIndexer_secretName() { return res }) + _ = someIndexer.IndexField(context.TODO(), &corev1.Pod{}, "spec.NodeName", func(o client.Object) []string { + nodeName := o.(*corev1.Pod).Spec.NodeName + if nodeName != "" { + return []string{nodeName} + } + return nil + }) + // elsewhere (e.g. in your reconciler) mySecretName := "someSecret" // derived from the reconcile.Request, for instance + myNode := "master-0" var podsWithSecrets corev1.PodList - _ = c.List(context.Background(), &podsWithSecrets, client.MatchingFields{"spec.volumes.secret.secretName": mySecretName}) + _ = c.List(context.Background(), &podsWithSecrets, client.MatchingFields{ + "spec.volumes.secret.secretName": mySecretName, + "spec.NodeName": myNode, + }) } diff --git a/pkg/client/fake/client.go b/pkg/client/fake/client.go index aaedac8440..62067cb19c 100644 --- a/pkg/client/fake/client.go +++ b/pkg/client/fake/client.go @@ -17,22 +17,32 @@ limitations under the License. package fake import ( - "bytes" "context" - "encoding/json" "errors" "fmt" "reflect" - "runtime/debug" - "strconv" "strings" "sync" "time" - // Using v4 to match upstream - jsonpatch "github.com/evanphx/json-patch" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - + /* + Stick with gopkg.in/evanphx/json-patch.v4 here to match + upstream Kubernetes code and avoid breaking changes introduced in v5. + - Kubernetes itself remains on json-patch v4 to avoid compatibility issues + tied to v5’s stricter RFC6902 compliance. + - The fake client code is adapted from client-go’s testing fixture, which also + relies on json-patch v4. + See: + https://github.com/kubernetes/kubernetes/pull/91622 (discussion of why K8s + stays on v4) + https://github.com/kubernetes/kubernetes/pull/120326 (v5.6.0+incompatible + missing a critical fix) + */ + + jsonpatch "gopkg.in/evanphx/json-patch.v4" + appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" @@ -40,42 +50,51 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/managedfields" utilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/strategicpatch" - "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apimachinery/pkg/watch" + clientgoapplyconfigurations "k8s.io/client-go/applyconfigurations" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/testing" - "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" "sigs.k8s.io/controller-runtime/pkg/internal/objectutil" ) -type versionedTracker struct { - testing.ObjectTracker - scheme *runtime.Scheme - withStatusSubresource sets.Set[schema.GroupVersionKind] -} - type fakeClient struct { - tracker versionedTracker - scheme *runtime.Scheme + // trackerWriteLock must be acquired before writing to + // the tracker or performing reads that affect a following + // write. + trackerWriteLock sync.Mutex + tracker versionedTracker + + schemeLock sync.RWMutex + scheme *runtime.Scheme + restMapper meta.RESTMapper withStatusSubresource sets.Set[schema.GroupVersionKind] // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. // The inner map maps from index name to IndexerFunc. indexes map[schema.GroupVersionKind]map[string]client.IndexerFunc + // indexesLock must be held when accessing indexes. + indexesLock sync.RWMutex - schemeWriteLock sync.Mutex + returnManagedFields bool } var _ client.WithWatch = &fakeClient{} @@ -84,25 +103,16 @@ const ( maxNameLength = 63 randomLength = 5 maxGeneratedNameLength = maxNameLength - randomLength + + subResourceScale = "scale" ) // NewFakeClient creates a new fake client for testing. // You can choose to initialize it with a slice of runtime.Object. -// -// Deprecated: Please use NewClientBuilder instead. func NewFakeClient(initObjs ...runtime.Object) client.WithWatch { return NewClientBuilder().WithRuntimeObjects(initObjs...).Build() } -// NewFakeClientWithScheme creates a new fake client with the given scheme -// for testing. -// You can choose to initialize it with a slice of runtime.Object. -// -// Deprecated: Please use NewClientBuilder instead. -func NewFakeClientWithScheme(clientScheme *runtime.Scheme, initObjs ...runtime.Object) client.WithWatch { - return NewClientBuilder().WithScheme(clientScheme).WithRuntimeObjects(initObjs...).Build() -} - // NewClientBuilder returns a new builder to create a fake client. func NewClientBuilder() *ClientBuilder { return &ClientBuilder{} @@ -118,6 +128,9 @@ type ClientBuilder struct { withStatusSubresource []client.Object objectTracker testing.ObjectTracker interceptorFuncs *interceptor.Funcs + typeConverters []managedfields.TypeConverter + returnManagedFields bool + isBuilt bool // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. // The inner map maps from index name to IndexerFunc. @@ -159,6 +172,8 @@ func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *C } // WithObjectTracker can be optionally used to initialize this fake client with testing.ObjectTracker. +// Setting this is incompatible with setting WithTypeConverters, as they are a setting on the +// tracker. func (f *ClientBuilder) WithObjectTracker(ot testing.ObjectTracker) *ClientBuilder { f.objectTracker = ot return f @@ -215,8 +230,36 @@ func (f *ClientBuilder) WithInterceptorFuncs(interceptorFuncs interceptor.Funcs) return f } +// WithTypeConverters sets the type converters for the fake client. The list is ordered and the first +// non-erroring converter is used. A type converter must be provided for all types the client is used +// for, otherwise it will error. +// +// This setting is incompatible with WithObjectTracker, as the type converters are a setting on the tracker. +// +// If unset, this defaults to: +// * clientgoapplyconfigurations.NewTypeConverter(scheme.Scheme), +// * managedfields.NewDeducedTypeConverter(), +// +// Be aware that the behavior of the `NewDeducedTypeConverter` might not match the behavior of the +// Kubernetes APIServer, it is recommended to provide a type converter for your types. TypeConverters +// are generated along with ApplyConfigurations. +func (f *ClientBuilder) WithTypeConverters(typeConverters ...managedfields.TypeConverter) *ClientBuilder { + f.typeConverters = append(f.typeConverters, typeConverters...) + return f +} + +// WithReturnManagedFields configures the fake client to return managedFields +// on objects. +func (f *ClientBuilder) WithReturnManagedFields() *ClientBuilder { + f.returnManagedFields = true + return f +} + // Build builds and returns a new fake client. func (f *ClientBuilder) Build() client.WithWatch { + if f.isBuilt { + panic("Build() must not be called multiple times when creating a ClientBuilder") + } if f.scheme == nil { f.scheme = scheme.Scheme } @@ -224,8 +267,6 @@ func (f *ClientBuilder) Build() client.WithWatch { f.restMapper = meta.NewDefaultRESTMapper([]schema.GroupVersion{}) } - var tracker versionedTracker - withStatusSubResource := sets.New(inTreeResourcesWithStatus()...) for _, o := range f.withStatusSubresource { gvk, err := apiutil.GVKForObject(o, f.scheme) @@ -235,10 +276,36 @@ func (f *ClientBuilder) Build() client.WithWatch { withStatusSubResource.Insert(gvk) } + if f.objectTracker != nil && len(f.typeConverters) > 0 { + panic(errors.New("WithObjectTracker and WithTypeConverters are incompatible")) + } + + var usesFieldManagedObjectTracker bool if f.objectTracker == nil { - tracker = versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme, withStatusSubresource: withStatusSubResource} - } else { - tracker = versionedTracker{ObjectTracker: f.objectTracker, scheme: f.scheme, withStatusSubresource: withStatusSubResource} + if len(f.typeConverters) == 0 { + // Use corresponding scheme to ensure the converter error + // for types it can't handle. + clientGoScheme := runtime.NewScheme() + if err := scheme.AddToScheme(clientGoScheme); err != nil { + panic(fmt.Sprintf("failed to construct client-go scheme: %v", err)) + } + f.typeConverters = []managedfields.TypeConverter{ + clientgoapplyconfigurations.NewTypeConverter(clientGoScheme), + managedfields.NewDeducedTypeConverter(), + } + } + f.objectTracker = testing.NewFieldManagedObjectTracker( + f.scheme, + serializer.NewCodecFactory(f.scheme).UniversalDecoder(), + multiTypeConverter{upstream: f.typeConverters}, + ) + usesFieldManagedObjectTracker = true + } + tracker := versionedTracker{ + upstream: f.objectTracker, + scheme: f.scheme, + withStatusSubresource: withStatusSubResource, + usesFieldManagedObjectTracker: usesFieldManagedObjectTracker, } for _, obj := range f.initObject { @@ -263,93 +330,31 @@ func (f *ClientBuilder) Build() client.WithWatch { restMapper: f.restMapper, indexes: f.indexes, withStatusSubresource: withStatusSubResource, + returnManagedFields: f.returnManagedFields, } if f.interceptorFuncs != nil { result = interceptor.NewClient(result, *f.interceptorFuncs) } + f.isBuilt = true return result } const trackerAddResourceVersion = "999" -func (t versionedTracker) Add(obj runtime.Object) error { - var objects []runtime.Object - if meta.IsListType(obj) { - var err error - objects, err = meta.ExtractList(obj) - if err != nil { - return err - } - } else { - objects = []runtime.Object{obj} - } - for _, obj := range objects { - accessor, err := meta.Accessor(obj) - if err != nil { - return fmt.Errorf("failed to get accessor for object: %w", err) - } - if accessor.GetDeletionTimestamp() != nil && len(accessor.GetFinalizers()) == 0 { - return fmt.Errorf("refusing to create obj %s with metadata.deletionTimestamp but no finalizers", accessor.GetName()) - } - if accessor.GetResourceVersion() == "" { - // We use a "magic" value of 999 here because this field - // is parsed as uint and and 0 is already used in Update. - // As we can't go lower, go very high instead so this can - // be recognized - accessor.SetResourceVersion(trackerAddResourceVersion) - } - - obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) - if err != nil { - return err - } - if err := t.ObjectTracker.Add(obj); err != nil { - return err - } - } - - return nil -} - -func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { - accessor, err := meta.Accessor(obj) - if err != nil { - return fmt.Errorf("failed to get accessor for object: %w", err) - } - if accessor.GetName() == "" { - return apierrors.NewInvalid( - obj.GetObjectKind().GroupVersionKind().GroupKind(), - accessor.GetName(), - field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) - } - if accessor.GetResourceVersion() != "" { - return apierrors.NewBadRequest("resourceVersion can not be set for Create requests") - } - accessor.SetResourceVersion("1") - obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) - if err != nil { - return err - } - if err := t.ObjectTracker.Create(gvr, obj, ns); err != nil { - accessor.SetResourceVersion("") - return err - } - - return nil -} - // convertFromUnstructuredIfNecessary will convert runtime.Unstructured for a GVK that is recognized // by the schema into the whatever the schema produces with New() for said GVK. // This is required because the tracker unconditionally saves on manipulations, but its List() implementation // tries to assign whatever it finds into a ListType it gets from schema.New() - Thus we have to ensure // we save as the very same type, otherwise subsequent List requests will fail. func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (runtime.Object, error) { - gvk := o.GetObjectKind().GroupVersionKind() - u, isUnstructured := o.(runtime.Unstructured) - if !isUnstructured || !s.Recognizes(gvk) { + if !isUnstructured { + return o, nil + } + gvk := o.GetObjectKind().GroupVersionKind() + if !s.Recognizes(gvk) { return o, nil } @@ -357,6 +362,9 @@ func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (ru if err != nil { return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", gvk, err) } + if _, isTypedUnstructured := typed.(runtime.Unstructured); isTypedUnstructured { + return o, nil + } unstructuredSerialized, err := json.Marshal(u) if err != nil { @@ -369,100 +377,18 @@ func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (ru return typed, nil } -func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { - isStatus := false - // We apply patches using a client-go reaction that ends up calling the trackers Update. As we can't change - // that reaction, we use the callstack to figure out if this originated from the status client. - if bytes.Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeSubResourceClient).Patch")) { - isStatus = true - } - return t.update(gvr, obj, ns, isStatus, false) -} - -func (t versionedTracker) update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, isStatus bool, deleting bool) error { - accessor, err := meta.Accessor(obj) - if err != nil { - return fmt.Errorf("failed to get accessor for object: %w", err) - } - - if accessor.GetName() == "" { - return apierrors.NewInvalid( - obj.GetObjectKind().GroupVersionKind().GroupKind(), - accessor.GetName(), - field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) - } - - gvk := obj.GetObjectKind().GroupVersionKind() - if gvk.Empty() { - gvk, err = apiutil.GVKForObject(obj, t.scheme) - if err != nil { - return err - } - } - - oldObject, err := t.ObjectTracker.Get(gvr, ns, accessor.GetName()) - if err != nil { - // If the resource is not found and the resource allows create on update, issue a - // create instead. - if apierrors.IsNotFound(err) && allowsCreateOnUpdate(gvk) { - return t.Create(gvr, obj, ns) - } - return err - } - - if t.withStatusSubresource.Has(gvk) { - if isStatus { // copy everything but status and metadata.ResourceVersion from original object - if err := copyNonStatusFrom(oldObject, obj); err != nil { - return fmt.Errorf("failed to copy non-status field for object with status subresouce: %w", err) - } - } else { // copy status from original object - if err := copyStatusFrom(oldObject, obj); err != nil { - return fmt.Errorf("failed to copy the status for object with status subresource: %w", err) - } - } - } else if isStatus { - return apierrors.NewNotFound(gvr.GroupResource(), accessor.GetName()) - } - - oldAccessor, err := meta.Accessor(oldObject) - if err != nil { +func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { return err } - // If the new object does not have the resource version set and it allows unconditional update, - // default it to the resource version of the existing resource - if accessor.GetResourceVersion() == "" && allowsUnconditionalUpdate(gvk) { - accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) - } - if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() { - return apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified")) - } - if oldAccessor.GetResourceVersion() == "" { - oldAccessor.SetResourceVersion("0") - } - intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64) - if err != nil { - return fmt.Errorf("can not convert resourceVersion %q to int: %w", oldAccessor.GetResourceVersion(), err) - } - intResourceVersion++ - accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10)) - - if !deleting && !deletionTimestampEqual(accessor, oldAccessor) { - return fmt.Errorf("error: Unable to edit %s: metadata.deletionTimestamp field is immutable", accessor.GetName()) - } - - if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 { - return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) - } - obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvr, err := getGVRFromObject(obj, c.scheme) if err != nil { return err } - return t.ObjectTracker.Update(gvr, obj, ns) -} - -func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - gvr, err := getGVRFromObject(obj, c.scheme) + gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err } @@ -471,28 +397,41 @@ func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.O return err } - gvk, err := apiutil.GVKForObject(obj, c.scheme) - if err != nil { - return err - } ta, err := meta.TypeAccessor(o) if err != nil { return err } - ta.SetKind(gvk.Kind) + + // If the final object is unstructuctured, the json + // representation must contain GVK or the apimachinery + // json serializer will error out. ta.SetAPIVersion(gvk.GroupVersion().String()) + ta.SetKind(gvk.Kind) j, err := json.Marshal(o) if err != nil { return err } - decoder := scheme.Codecs.UniversalDecoder() zero(obj) - _, _, err = decoder.Decode(j, nil, obj) - return err + if err := json.Unmarshal(j, obj); err != nil { + return err + } + + if !c.returnManagedFields { + obj.SetManagedFields(nil) + } + + return ensureTypeMeta(obj, gvk) } func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(list); err != nil { + return nil, err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvk, err := apiutil.GVKForObject(list, c.scheme) if err != nil { return nil, err @@ -508,21 +447,30 @@ func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ... } func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err } - originalKind := gvk.Kind - + originalGVK := gvk gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + listGVK := gvk + listGVK.Kind += "List" - if _, isUnstructuredList := obj.(runtime.Unstructured); isUnstructuredList && !c.scheme.Recognizes(gvk) { + if _, isUnstructuredList := obj.(runtime.Unstructured); isUnstructuredList && !c.scheme.Recognizes(listGVK) { // We need to register the ListKind with UnstructuredList: // https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51 - c.schemeWriteLock.Lock() + c.schemeLock.RUnlock() + c.schemeLock.Lock() c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) - c.schemeWriteLock.Unlock() + c.schemeLock.Unlock() + c.schemeLock.RLock() } listOpts := client.ListOptions{} @@ -534,35 +482,40 @@ func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...cl return err } - ta, err := meta.TypeAccessor(o) - if err != nil { - return err - } - ta.SetKind(originalKind) - ta.SetAPIVersion(gvk.GroupVersion().String()) - j, err := json.Marshal(o) if err != nil { return err } - decoder := scheme.Codecs.UniversalDecoder() zero(obj) - _, _, err = decoder.Decode(j, nil, obj) + if err := ensureTypeMeta(obj, originalGVK); err != nil { + return err + } + objCopy := obj.DeepCopyObject().(client.ObjectList) + if err := json.Unmarshal(j, objCopy); err != nil { + return err + } + + objs, err := meta.ExtractList(objCopy) if err != nil { return err } + for _, o := range objs { + if err := ensureTypeMeta(o, gvk); err != nil { + return err + } + + if !c.returnManagedFields { + o.(metav1.Object).SetManagedFields(nil) + } + } + if listOpts.LabelSelector == nil && listOpts.FieldSelector == nil { - return nil + return meta.SetList(obj, objs) } // If we're here, either a label or field selector are specified (or both), so before we return // the list we must filter it. If both selectors are set, they are ANDed. - objs, err := meta.ExtractList(obj) - if err != nil { - return err - } - filteredList, err := c.filterList(objs, gvk, listOpts.LabelSelector, listOpts.FieldSelector) if err != nil { return err @@ -595,26 +548,34 @@ func (c *fakeClient) filterList(list []runtime.Object, gvk schema.GroupVersionKi } func (c *fakeClient) filterWithFields(list []runtime.Object, gvk schema.GroupVersionKind, fs fields.Selector) ([]runtime.Object, error) { - // We only allow filtering on the basis of a single field to ensure consistency with the - // behavior of the cache reader (which we're faking here). - fieldKey, fieldVal, requiresExact := selector.RequiresExactMatch(fs) + requiresExact := selector.RequiresExactMatch(fs) if !requiresExact { - return nil, fmt.Errorf("field selector %s is not in one of the two supported forms \"key==val\" or \"key=val\"", - fs) + return nil, fmt.Errorf(`field selector %s is not in one of the two supported forms "key==val" or "key=val"`, fs) } + c.indexesLock.RLock() + defer c.indexesLock.RUnlock() // Field selection is mimicked via indexes, so there's no sane answer this function can give // if there are no indexes registered for the GroupVersionKind of the objects in the list. indexes := c.indexes[gvk] - if len(indexes) == 0 || indexes[fieldKey] == nil { - return nil, fmt.Errorf("List on GroupVersionKind %v specifies selector on field %s, but no "+ - "index with name %s has been registered for GroupVersionKind %v", gvk, fieldKey, fieldKey, gvk) + for _, req := range fs.Requirements() { + if len(indexes) == 0 || indexes[req.Field] == nil { + return nil, fmt.Errorf("List on GroupVersionKind %v specifies selector on field %s, but no "+ + "index with name %s has been registered for GroupVersionKind %v", gvk, req.Field, req.Field, gvk) + } } - indexExtractor := indexes[fieldKey] filteredList := make([]runtime.Object, 0, len(list)) for _, obj := range list { - if c.objMatchesFieldSelector(obj, indexExtractor, fieldVal) { + matches := true + for _, req := range fs.Requirements() { + indexExtractor := indexes[req.Field] + if !c.objMatchesFieldSelector(obj, indexExtractor, req.Value) { + matches = false + break + } + } + if matches { filteredList = append(filteredList, obj) } } @@ -655,6 +616,13 @@ func (c *fakeClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { } func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + createOptions := &client.CreateOptions{} createOptions.ApplyOptions(opts) @@ -685,10 +653,35 @@ func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...clie accessor.SetDeletionTimestamp(nil) } - return c.tracker.Create(gvr, obj, accessor.GetNamespace()) + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + + if err := c.tracker.Create(gvr, obj, accessor.GetNamespace(), *createOptions.AsCreateOptions()); err != nil { + // The managed fields tracker sets gvk even on errors + _ = ensureTypeMeta(obj, gvk) + return err + } + + if !c.returnManagedFields { + obj.SetManagedFields(nil) + } + + return ensureTypeMeta(obj, gvk) } func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvr, err := getGVRFromObject(obj, c.scheme) if err != nil { return err @@ -706,6 +699,8 @@ func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie } } + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() // Check the ResourceVersion if that Precondition was specified. if delOptions.Preconditions != nil && delOptions.Preconditions.ResourceVersion != nil { name := accessor.GetName() @@ -728,10 +723,17 @@ func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie } } - return c.deleteObject(gvr, accessor) + return c.deleteObjectLocked(gvr, accessor) } func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err @@ -746,6 +748,9 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts .. } } + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + gvr, _ := meta.UnsafeGuessKindToResource(gvk) o, err := c.tracker.List(gvr, gvk, dcOptions.Namespace) if err != nil { @@ -765,7 +770,7 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts .. if err != nil { return err } - err = c.deleteObject(gvr, accessor) + err = c.deleteObjectLocked(gvr, accessor) if err != nil { return err } @@ -778,6 +783,13 @@ func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...clie } func (c *fakeClient) update(obj client.Object, isStatus bool, opts ...client.UpdateOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + updateOptions := &client.UpdateOptions{} updateOptions.ApplyOptions(opts) @@ -791,21 +803,111 @@ func (c *fakeClient) update(obj client.Object, isStatus bool, opts ...client.Upd if err != nil { return err } + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } accessor, err := meta.Accessor(obj) if err != nil { return err } - return c.tracker.update(gvr, obj, accessor.GetNamespace(), isStatus, false) + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + + // Retain managed fields + // We can ignore all errors here since update will fail if we encounter an error. + obj.SetManagedFields(nil) + current, _ := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) + if currentMetaObj, ok := current.(metav1.Object); ok { + obj.SetManagedFields(currentMetaObj.GetManagedFields()) + } + + if err := c.tracker.update(gvr, obj, accessor.GetNamespace(), isStatus, false, *updateOptions.AsUpdateOptions()); err != nil { + return err + } + + if !c.returnManagedFields { + obj.SetManagedFields(nil) + } + + return ensureTypeMeta(obj, gvk) } func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { return c.patch(obj, patch, opts...) } +func (c *fakeClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + applyOpts := &client.ApplyOptions{} + applyOpts.ApplyOptions(opts) + + data, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("failed to marshal apply configuration: %w", err) + } + + u := &unstructured.Unstructured{} + if err := json.Unmarshal(data, u); err != nil { + return fmt.Errorf("failed to unmarshal apply configuration: %w", err) + } + + applyPatch := &fakeApplyPatch{} + + patchOpts := &client.PatchOptions{} + patchOpts.Raw = applyOpts.AsPatchOptions() + + if err := c.patch(u, applyPatch, patchOpts); err != nil { + return err + } + + acJSON, err := json.Marshal(u) + if err != nil { + return fmt.Errorf("failed to marshal patched object: %w", err) + } + + // We have to zero the object in case it contained a status and there is a + // status subresource. If its the private `unstructuredApplyConfiguration` + // we can not zero all of it, as that will cause the embedded Unstructured + // to be nil which then causes a NPD in the json.Unmarshal below. + switch reflect.TypeOf(obj).String() { + case "*client.unstructuredApplyConfiguration": + zero(reflect.ValueOf(obj).Elem().FieldByName("Unstructured").Interface()) + default: + zero(obj) + } + if err := json.Unmarshal(acJSON, obj); err != nil { + return fmt.Errorf("failed to unmarshal patched object: %w", err) + } + + return nil +} + +type fakeApplyPatch struct{} + +func (p *fakeApplyPatch) Type() types.PatchType { + return types.ApplyPatchType +} + +func (p *fakeApplyPatch) Data(obj client.Object) ([]byte, error) { + return json.Marshal(obj) +} + func (c *fakeClient) patch(obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := c.addToSchemeIfUnknownAndUnstructuredOrPartial(obj); err != nil { + return err + } + patchOptions := &client.PatchOptions{} patchOptions.ApplyOptions(opts) + if errs := validation.ValidatePatchOptions(patchOptions.AsPatchOptions(), patch.Type()); len(errs) > 0 { + return apierrors.NewInvalid(schema.GroupKind{Group: "meta.k8s.io", Kind: "PatchOptions"}, "", errs) + } + + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + for _, dryRunOpt := range patchOptions.DryRun { if dryRunOpt == metav1.DryRunAll { return nil @@ -816,74 +918,125 @@ func (c *fakeClient) patch(obj client.Object, patch client.Patch, opts ...client if err != nil { return err } - accessor, err := meta.Accessor(obj) - if err != nil { - return err - } - data, err := patch.Data(obj) + gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return err } - - gvk, err := apiutil.GVKForObject(obj, c.scheme) + accessor, err := meta.Accessor(obj) if err != nil { return err } + var isApplyCreate bool + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() oldObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) if err != nil { - return err + if !apierrors.IsNotFound(err) || patch.Type() != types.ApplyPatchType { + return err + } + oldObj = &unstructured.Unstructured{} + isApplyCreate = true } oldAccessor, err := meta.Accessor(oldObj) if err != nil { return err } - // Apply patch without updating object. - // To remain in accordance with the behavior of k8s api behavior, - // a patch must not allow for changes to the deletionTimestamp of an object. - // The reaction() function applies the patch to the object and calls Update(), - // whereas dryPatch() replicates this behavior but skips the call to Update(). - // This ensures that the patch may be rejected if a deletionTimestamp is modified, prior - // to updating the object. - action := testing.NewPatchAction(gvr, accessor.GetNamespace(), accessor.GetName(), patch.Type(), data) - o, err := dryPatch(action, c.tracker) - if err != nil { - return err + if patch.Type() == types.ApplyPatchType { + if isApplyCreate { + // Overwrite it unconditionally, this matches the apiserver behavior + // which allows to set it on create, but will then ignore it. + obj.SetResourceVersion("1") + } else { + // SSA deletionTimestamp updates are silently ignored + obj.SetDeletionTimestamp(oldAccessor.GetDeletionTimestamp()) + } } - newObj, err := meta.Accessor(o) + + data, err := patch.Data(obj) if err != nil { return err } - // Validate that deletionTimestamp has not been changed - if !deletionTimestampEqual(newObj, oldAccessor) { - return fmt.Errorf("rejected patch, metadata.deletionTimestamp immutable") + action := testing.NewPatchActionWithOptions( + gvr, + accessor.GetNamespace(), + accessor.GetName(), + patch.Type(), + data, + *patchOptions.AsPatchOptions(), + ) + + // Apply is implemented in the tracker and calling it has side-effects + // such as bumping RV and updating managedFields timestamps, hence we + // can not dry-run it. Luckily, the only validation we use it for + // doesn't apply to SSA - Creating objects with non-nil deletionTimestamp + // through SSA is possible and updating the deletionTimestamp is valid, + // but has no effect. + if patch.Type() != types.ApplyPatchType { + // Apply patch without updating object. + // To remain in accordance with the behavior of k8s api behavior, + // a patch must not allow for changes to the deletionTimestamp of an object. + // The reaction() function applies the patch to the object and calls Update(), + // whereas dryPatch() replicates this behavior but skips the call to Update(). + // This ensures that the patch may be rejected if a deletionTimestamp is modified, prior + // to updating the object. + o, err := dryPatch(action, c.tracker) + if err != nil { + return err + } + newObj, err := meta.Accessor(o) + if err != nil { + return err + } + + // Validate that deletionTimestamp has not been changed + if !deletionTimestampEqual(newObj, oldAccessor) { + return fmt.Errorf("rejected patch, metadata.deletionTimestamp immutable") + } } reaction := testing.ObjectReaction(c.tracker) handled, o, err := reaction(action) if err != nil { + // The reaction calls tracker.Get after tracker.Apply to return the object, + // but we may have deleted it in tracker.Apply if there was no finalizer + // left. + if apierrors.IsNotFound(err) && + patch.Type() == types.ApplyPatchType && + oldAccessor.GetDeletionTimestamp() != nil && + len(obj.GetFinalizers()) == 0 { + return nil + } return err } if !handled { panic("tracker could not handle patch method") } + ta, err := meta.TypeAccessor(o) if err != nil { return err } - ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + ta.SetKind(gvk.Kind) j, err := json.Marshal(o) if err != nil { return err } - decoder := scheme.Codecs.UniversalDecoder() zero(obj) - _, _, err = decoder.Decode(j, nil, obj) - return err + if err := json.Unmarshal(j, obj); err != nil { + return err + } + + if !c.returnManagedFields { + obj.SetManagedFields(nil) + } + + return ensureTypeMeta(obj, gvk) } // Applying a patch results in a deletionTimestamp that is truncated to the nearest second. @@ -911,6 +1064,9 @@ func dryPatch(action testing.PatchActionImpl, tracker testing.ObjectTracker) (ru obj, err := tracker.Get(gvr, ns, action.GetName()) if err != nil { + if apierrors.IsNotFound(err) && action.GetPatchType() == types.ApplyPatchType { + return &unstructured.Unstructured{}, nil + } return nil, err } @@ -947,7 +1103,7 @@ func dryPatch(action testing.PatchActionImpl, tracker testing.ObjectTracker) (ru if err := json.Unmarshal(modified, obj); err != nil { return nil, err } - case types.StrategicMergePatchType, types.ApplyPatchType: + case types.StrategicMergePatchType: mergedByte, err := strategicpatch.StrategicMergePatch(old, action.GetPatch(), obj) if err != nil { return nil, err @@ -955,65 +1111,43 @@ func dryPatch(action testing.PatchActionImpl, tracker testing.ObjectTracker) (ru if err = json.Unmarshal(mergedByte, obj); err != nil { return nil, err } + case types.ApplyCBORPatchType: + return nil, errors.New("apply CBOR patches are not supported in the fake client") + case types.ApplyPatchType: + return nil, errors.New("bug in controller-runtime: should not end up in dryPatch for SSA") default: - return nil, fmt.Errorf("PatchType is not supported") + return nil, fmt.Errorf("%s PatchType is not supported", action.GetPatchType()) } return obj, nil } -func copyNonStatusFrom(old, new runtime.Object) error { - newClientObject, ok := new.(client.Object) - if !ok { - return fmt.Errorf("%T is not a client.Object", new) - } - // The only thing other than status we have to retain - rv := newClientObject.GetResourceVersion() - +// copyStatusFrom copies the status from old into new +func copyStatusFrom(old, n runtime.Object) error { oldMapStringAny, err := toMapStringAny(old) if err != nil { return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err) } - newMapStringAny, err := toMapStringAny(new) + newMapStringAny, err := toMapStringAny(n) if err != nil { return fmt.Errorf("failed to convert new to *unststructured.Unstructured: %w", err) } - // delete everything other than status in case it has fields that were not present in - // the old object - for k := range newMapStringAny { - if k != "status" { - delete(newMapStringAny, k) - } - } - // copy everything other than status from the old object - for k := range oldMapStringAny { - if k != "status" { - newMapStringAny[k] = oldMapStringAny[k] - } - } + newMapStringAny["status"] = oldMapStringAny["status"] - if err := fromMapStringAny(newMapStringAny, new); err != nil { + if err := fromMapStringAny(newMapStringAny, n); err != nil { return fmt.Errorf("failed to convert back from map[string]any: %w", err) } - newClientObject.SetResourceVersion(rv) return nil } -// copyStatusFrom copies the status from old into new -func copyStatusFrom(old, new runtime.Object) error { +// copyFrom copies from old into new +func copyFrom(old, n runtime.Object) error { oldMapStringAny, err := toMapStringAny(old) if err != nil { return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err) } - newMapStringAny, err := toMapStringAny(new) - if err != nil { - return fmt.Errorf("failed to convert new to *unststructured.Unstructured: %w", err) - } - - newMapStringAny["status"] = oldMapStringAny["status"] - - if err := fromMapStringAny(newMapStringAny, new); err != nil { + if err := fromMapStringAny(oldMapStringAny, n); err != nil { return fmt.Errorf("failed to convert back from map[string]any: %w", err) } @@ -1045,6 +1179,7 @@ func fromMapStringAny(u map[string]any, target runtime.Object) error { return fmt.Errorf("failed to serialize: %w", err) } + zero(target) if err := json.Unmarshal(serialized, &target); err != nil { return fmt.Errorf("failed to deserialize: %w", err) } @@ -1060,7 +1195,7 @@ func (c *fakeClient) SubResource(subResource string) client.SubResourceClient { return &fakeSubResourceClient{client: c, subResource: subResource} } -func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor metav1.Object) error { +func (c *fakeClient) deleteObjectLocked(gvr schema.GroupVersionResource, accessor metav1.Object) error { old, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) if err == nil { oldAccessor, err := meta.Accessor(old) @@ -1070,12 +1205,12 @@ func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor meta oldAccessor.SetDeletionTimestamp(&now) // Call update directly with mutability parameter set to true to allow // changes to deletionTimestamp - return c.tracker.update(gvr, old, accessor.GetNamespace(), false, true) + return c.tracker.update(gvr, old, accessor.GetNamespace(), false, true, metav1.UpdateOptions{}) } } } - //TODO: implement propagation + // TODO: implement propagation return c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) } @@ -1094,7 +1229,26 @@ type fakeSubResourceClient struct { } func (sw *fakeSubResourceClient) Get(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { - panic("fakeSubResourceClient does not support get") + switch sw.subResource { + case subResourceScale: + // Actual client looks up resource, then extracts the scale sub-resource: + // https://github.com/kubernetes/kubernetes/blob/fb6bbc9781d11a87688c398778525c4e1dcb0f08/pkg/registry/apps/deployment/storage/storage.go#L307 + if err := sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { + return err + } + scale, isScale := subResource.(*autoscalingv1.Scale) + if !isScale { + return apierrors.NewBadRequest(fmt.Sprintf("expected Scale, got %T", subResource)) + } + scaleOut, err := extractScale(obj) + if err != nil { + return err + } + *scale = *scaleOut + return nil + default: + return fmt.Errorf("fakeSubResourceClient does not support get for %s", sw.subResource) + } } func (sw *fakeSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { @@ -1105,13 +1259,26 @@ func (sw *fakeSubResourceClient) Create(ctx context.Context, obj client.Object, _, isEviction = subResource.(*policyv1.Eviction) } if !isEviction { - return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %t, expected Eviction", subResource)) + return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %T, expected Eviction", subResource)) } if _, isPod := obj.(*corev1.Pod); !isPod { return apierrors.NewNotFound(schema.GroupResource{}, "") } return sw.client.Delete(ctx, obj) + case "token": + tokenRequest, isTokenRequest := subResource.(*authenticationv1.TokenRequest) + if !isTokenRequest { + return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %T, expected TokenRequest", subResource)) + } + if _, isServiceAccount := obj.(*corev1.ServiceAccount); !isServiceAccount { + return apierrors.NewNotFound(schema.GroupResource{}, "") + } + + tokenRequest.Status.Token = "fake-token" + tokenRequest.Status.ExpirationTimestamp = metav1.Date(6041, 1, 1, 0, 0, 0, 0, time.UTC) + + return sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj) default: return fmt.Errorf("fakeSubResourceWriter does not support create for %s", sw.subResource) } @@ -1121,11 +1288,30 @@ func (sw *fakeSubResourceClient) Update(ctx context.Context, obj client.Object, updateOptions := client.SubResourceUpdateOptions{} updateOptions.ApplyOptions(opts) - body := obj - if updateOptions.SubResourceBody != nil { - body = updateOptions.SubResourceBody + switch sw.subResource { + case subResourceScale: + if err := sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj.DeepCopyObject().(client.Object)); err != nil { + return err + } + if updateOptions.SubResourceBody == nil { + return apierrors.NewBadRequest("missing SubResourceBody") + } + + scale, isScale := updateOptions.SubResourceBody.(*autoscalingv1.Scale) + if !isScale { + return apierrors.NewBadRequest(fmt.Sprintf("expected Scale, got %T", updateOptions.SubResourceBody)) + } + if err := applyScale(obj, scale); err != nil { + return err + } + return sw.client.update(obj, false, &updateOptions.UpdateOptions) + default: + body := obj + if updateOptions.SubResourceBody != nil { + body = updateOptions.SubResourceBody + } + return sw.client.update(body, true, &updateOptions.UpdateOptions) } - return sw.client.update(body, true, &updateOptions.UpdateOptions) } func (sw *fakeSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { @@ -1137,9 +1323,54 @@ func (sw *fakeSubResourceClient) Patch(ctx context.Context, obj client.Object, p body = patchOptions.SubResourceBody } + // this is necessary to identify that last call was made for status patch, through stack trace. + if sw.subResource == "status" { + return sw.statusPatch(body, patch, patchOptions) + } + + return sw.client.patch(body, patch, &patchOptions.PatchOptions) +} + +func (sw *fakeSubResourceClient) statusPatch(body client.Object, patch client.Patch, patchOptions client.SubResourcePatchOptions) error { return sw.client.patch(body, patch, &patchOptions.PatchOptions) } +func (sw *fakeSubResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + if sw.subResource != "status" { + return errors.New("fakeSubResourceClient currently only supports Apply for status subresource") + } + + applyOpts := &client.SubResourceApplyOptions{} + applyOpts.ApplyOpts(opts) + + data, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("failed to marshal apply configuration: %w", err) + } + + u := &unstructured.Unstructured{} + if err := json.Unmarshal(data, u); err != nil { + return fmt.Errorf("failed to unmarshal apply configuration: %w", err) + } + + patchOpts := &client.SubResourcePatchOptions{} + patchOpts.Raw = applyOpts.AsPatchOptions() + + if applyOpts.SubResourceBody != nil { + subResourceBodySerialized, err := json.Marshal(applyOpts.SubResourceBody) + if err != nil { + return fmt.Errorf("failed to serialize subresource body: %w", err) + } + subResourceBody := &unstructured.Unstructured{} + if err := json.Unmarshal(subResourceBodySerialized, subResourceBody); err != nil { + return fmt.Errorf("failed to unmarshal subresource body: %w", err) + } + patchOpts.SubResourceBody = subResourceBody + } + + return sw.Patch(ctx, u, &fakeApplyPatch{}, patchOpts) +} + func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool { switch gvk.Group { case "apps": @@ -1177,7 +1408,7 @@ func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool { case "PodSecurityPolicy": return true } - case "rbac": + case "rbac.authorization.k8s.io": switch gvk.Kind { case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding": return true @@ -1270,6 +1501,8 @@ func inTreeResourcesWithStatus() []schema.GroupVersionKind { {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "FlowSchema"}, {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "PriorityLevelConfiguration"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1", Kind: "FlowSchema"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1", Kind: "PriorityLevelConfiguration"}, } } @@ -1281,3 +1514,216 @@ func zero(x interface{}) { res := reflect.ValueOf(x).Elem() res.Set(reflect.Zero(res.Type())) } + +// getSingleOrZeroOptions returns the single options value in the slice, its +// zero value if the slice is empty, or an error if the slice contains more than +// one option value. +func getSingleOrZeroOptions[T any](opts []T) (opt T, err error) { + switch len(opts) { + case 0: + case 1: + opt = opts[0] + default: + err = fmt.Errorf("expected single or no options value, got %d values", len(opts)) + } + return +} + +func extractScale(obj client.Object) (*autoscalingv1.Scale, error) { + switch obj := obj.(type) { + case *appsv1.Deployment: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + case *appsv1.ReplicaSet: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + case *corev1.ReplicationController: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: labels.Set(obj.Spec.Selector).String(), + }, + }, nil + case *appsv1.StatefulSet: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + default: + // TODO: CRDs https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource + return nil, fmt.Errorf("unimplemented scale subresource for resource %T", obj) + } +} + +func applyScale(obj client.Object, scale *autoscalingv1.Scale) error { + switch obj := obj.(type) { + case *appsv1.Deployment: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *appsv1.ReplicaSet: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *corev1.ReplicationController: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *appsv1.StatefulSet: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + default: + // TODO: CRDs https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource + return fmt.Errorf("unimplemented scale subresource for resource %T", obj) + } + return nil +} + +// AddIndex adds an index to a fake client. It will panic if used with a client that is not a fake client. +// It will error if there is already an index for given object with the same name as field. +// +// It can be used to test code that adds indexes to the cache at runtime. +func AddIndex(c client.Client, obj runtime.Object, field string, extractValue client.IndexerFunc) error { + fakeClient, isFakeClient := c.(*fakeClient) + if !isFakeClient { + panic("AddIndex can only be used with a fake client") + } + fakeClient.indexesLock.Lock() + defer fakeClient.indexesLock.Unlock() + + if fakeClient.indexes == nil { + fakeClient.indexes = make(map[schema.GroupVersionKind]map[string]client.IndexerFunc, 1) + } + + gvk, err := apiutil.GVKForObject(obj, fakeClient.scheme) + if err != nil { + return fmt.Errorf("failed to get gvk for %T: %w", obj, err) + } + + if fakeClient.indexes[gvk] == nil { + fakeClient.indexes[gvk] = make(map[string]client.IndexerFunc, 1) + } + + if fakeClient.indexes[gvk][field] != nil { + return fmt.Errorf("index %s already exists", field) + } + + fakeClient.indexes[gvk][field] = extractValue + + return nil +} + +func (c *fakeClient) addToSchemeIfUnknownAndUnstructuredOrPartial(obj runtime.Object) error { + c.schemeLock.Lock() + defer c.schemeLock.Unlock() + + _, isUnstructured := obj.(*unstructured.Unstructured) + _, isUnstructuredList := obj.(*unstructured.UnstructuredList) + _, isPartial := obj.(*metav1.PartialObjectMetadata) + _, isPartialList := obj.(*metav1.PartialObjectMetadataList) + if !isUnstructured && !isUnstructuredList && !isPartial && !isPartialList { + return nil + } + + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + if !c.scheme.Recognizes(gvk) { + c.scheme.AddKnownTypeWithName(gvk, obj) + } + + return nil +} + +func ensureTypeMeta(obj runtime.Object, gvk schema.GroupVersionKind) error { + ta, err := meta.TypeAccessor(obj) + if err != nil { + return err + } + _, isUnstructured := obj.(runtime.Unstructured) + _, isPartialObject := obj.(*metav1.PartialObjectMetadata) + _, isPartialObjectList := obj.(*metav1.PartialObjectMetadataList) + if !isUnstructured && !isPartialObject && !isPartialObjectList { + ta.SetKind("") + ta.SetAPIVersion("") + return nil + } + + ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + + return nil +} diff --git a/pkg/client/fake/client_test.go b/pkg/client/fake/client_test.go index 8fd27f29e9..36722b4ddc 100644 --- a/pkg/client/fake/client_test.go +++ b/pkg/client/fake/client_test.go @@ -21,29 +21,45 @@ import ( "encoding/json" "fmt" "strconv" + "sync" "time" - "sigs.k8s.io/controller-runtime/pkg/client/interceptor" - "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes/fake" - appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" + clientgoapplyconfigurations "k8s.io/client-go/applyconfigurations" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + "k8s.io/client-go/kubernetes/fake" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +const ( + machineIDFromStatusUpdate = "machine-id-from-status-update" + cidrFromStatusUpdate = "cidr-from-status-update" ) var _ = Describe("Fake client", func() { @@ -55,10 +71,6 @@ var _ = Describe("Fake client", func() { BeforeEach(func() { replicas := int32(1) dep = &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "Deployment", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", Namespace: "ns1", @@ -72,10 +84,6 @@ var _ = Describe("Fake client", func() { }, } dep2 = &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "Deployment", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment-2", Namespace: "ns1", @@ -89,10 +97,6 @@ var _ = Describe("Fake client", func() { }, } cm = &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-cm", Namespace: "ns2", @@ -105,19 +109,19 @@ var _ = Describe("Fake client", func() { }) AssertClientWithoutIndexBehavior := func() { - It("should be able to Get", func() { + It("should be able to Get", func(ctx SpecContext) { By("Getting a deployment") namespacedName := types.NamespacedName{ Name: "test-deployment", Namespace: "ns1", } obj := &appsv1.Deployment{} - err := cl.Get(context.Background(), namespacedName, obj) + err := cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(dep)) }) - It("should be able to Get using unstructured", func() { + It("should be able to Get using unstructured", func(ctx SpecContext) { By("Getting a deployment") namespacedName := types.NamespacedName{ Name: "test-deployment", @@ -126,47 +130,53 @@ var _ = Describe("Fake client", func() { obj := &unstructured.Unstructured{} obj.SetAPIVersion("apps/v1") obj.SetKind("Deployment") - err := cl.Get(context.Background(), namespacedName, obj) + err := cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) }) - It("should be able to List", func() { + It("should be able to List", func(ctx SpecContext) { By("Listing all deployments in a namespace") list := &appsv1.DeploymentList{} - err := cl.List(context.Background(), list, client.InNamespace("ns1")) + err := cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(2)) Expect(list.Items).To(ConsistOf(*dep, *dep2)) }) - It("should be able to List using unstructured list", func() { + It("should be able to List using unstructured list", func(ctx SpecContext) { By("Listing all deployments in a namespace") list := &unstructured.UnstructuredList{} list.SetAPIVersion("apps/v1") list.SetKind("DeploymentList") - err := cl.List(context.Background(), list, client.InNamespace("ns1")) + err := cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("apps/v1")) + Expect(list.GetKind()).To(Equal("DeploymentList")) Expect(list.Items).To(HaveLen(2)) }) - It("should be able to List using unstructured list when setting a non-list kind", func() { + It("should be able to List using unstructured list when setting a non-list kind", func(ctx SpecContext) { By("Listing all deployments in a namespace") list := &unstructured.UnstructuredList{} list.SetAPIVersion("apps/v1") list.SetKind("Deployment") - err := cl.List(context.Background(), list, client.InNamespace("ns1")) + err := cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("apps/v1")) + Expect(list.GetKind()).To(Equal("Deployment")) Expect(list.Items).To(HaveLen(2)) }) - It("should be able to retrieve registered objects that got manipulated as unstructured", func() { + It("should be able to retrieve registered objects that got manipulated as unstructured", func(ctx SpecContext) { list := func() { By("Listing all endpoints in a namespace") list := &unstructured.UnstructuredList{} list.SetAPIVersion("v1") list.SetKind("EndpointsList") - err := cl.List(context.Background(), list, client.InNamespace("ns1")) + err := cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("v1")) + Expect(list.GetKind()).To(Equal("EndpointsList")) Expect(list.Items).To(HaveLen(1)) } @@ -180,46 +190,46 @@ var _ = Describe("Fake client", func() { } By("Adding the object during client initialization") - cl = NewFakeClient(unstructuredEndpoint()) + cl = NewClientBuilder().WithRuntimeObjects(unstructuredEndpoint()).Build() list() - Expect(cl.Delete(context.Background(), unstructuredEndpoint())).To(Succeed()) + Expect(cl.Delete(ctx, unstructuredEndpoint())).To(Succeed()) By("Creating an object") item := unstructuredEndpoint() - err := cl.Create(context.Background(), item) + err := cl.Create(ctx, item) Expect(err).ToNot(HaveOccurred()) list() By("Updating the object") item.SetAnnotations(map[string]string{"foo": "bar"}) - err = cl.Update(context.Background(), item) + err = cl.Update(ctx, item) Expect(err).ToNot(HaveOccurred()) list() By("Patching the object") old := item.DeepCopy() item.SetAnnotations(map[string]string{"bar": "baz"}) - err = cl.Patch(context.Background(), item, client.MergeFrom(old)) + err = cl.Patch(ctx, item, client.MergeFrom(old)) Expect(err).ToNot(HaveOccurred()) list() }) - It("should be able to Create an unregistered type using unstructured", func() { + It("should be able to Create an unregistered type using unstructured", func(ctx SpecContext) { item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v1") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) + err := cl.Create(ctx, item) Expect(err).ToNot(HaveOccurred()) }) - It("should be able to Get an unregisted type using unstructured", func() { + It("should be able to Get an unregisted type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v2") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) + err := cl.Create(ctx, item) Expect(err).ToNot(HaveOccurred()) By("Getting and the object") @@ -227,39 +237,43 @@ var _ = Describe("Fake client", func() { item.SetAPIVersion("custom/v2") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) Expect(err).ToNot(HaveOccurred()) }) - It("should be able to List an unregistered type using unstructured", func() { + It("should be able to List an unregistered type using unstructured with ListKind", func(ctx SpecContext) { list := &unstructured.UnstructuredList{} list.SetAPIVersion("custom/v3") list.SetKind("ImageList") - err := cl.List(context.Background(), list) + err := cl.List(ctx, list) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("custom/v3")) + Expect(list.GetKind()).To(Equal("ImageList")) Expect(err).ToNot(HaveOccurred()) }) - It("should be able to List an unregistered type using unstructured", func() { + It("should be able to List an unregistered type using unstructured with Kind", func(ctx SpecContext) { list := &unstructured.UnstructuredList{} list.SetAPIVersion("custom/v4") list.SetKind("Image") - err := cl.List(context.Background(), list) + err := cl.List(ctx, list) Expect(err).ToNot(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("custom/v4")) + Expect(list.GetKind()).To(Equal("Image")) }) - It("should be able to Update an unregistered type using unstructured", func() { + It("should be able to Update an unregistered type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v5") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) + err := cl.Create(ctx, item) Expect(err).ToNot(HaveOccurred()) By("Updating the object") err = unstructured.SetNestedField(item.Object, int64(2), "spec", "replicas") Expect(err).ToNot(HaveOccurred()) - err = cl.Update(context.Background(), item) + err = cl.Update(ctx, item) Expect(err).ToNot(HaveOccurred()) By("Getting the object") @@ -267,7 +281,7 @@ var _ = Describe("Fake client", func() { item.SetAPIVersion("custom/v5") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) Expect(err).ToNot(HaveOccurred()) By("Inspecting the object") @@ -277,20 +291,20 @@ var _ = Describe("Fake client", func() { Expect(value).To(Equal(int64(2))) }) - It("should be able to Patch an unregistered type using unstructured", func() { + It("should be able to Patch an unregistered type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v6") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) + err := cl.Create(ctx, item) Expect(err).ToNot(HaveOccurred()) By("Updating the object") original := item.DeepCopy() err = unstructured.SetNestedField(item.Object, int64(2), "spec", "replicas") Expect(err).ToNot(HaveOccurred()) - err = cl.Patch(context.Background(), item, client.MergeFrom(original)) + err = cl.Patch(ctx, item, client.MergeFrom(original)) Expect(err).ToNot(HaveOccurred()) By("Getting the object") @@ -298,7 +312,7 @@ var _ = Describe("Fake client", func() { item.SetAPIVersion("custom/v6") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) Expect(err).ToNot(HaveOccurred()) By("Inspecting the object") @@ -308,17 +322,17 @@ var _ = Describe("Fake client", func() { Expect(value).To(Equal(int64(2))) }) - It("should be able to Delete an unregistered type using unstructured", func() { + It("should be able to Delete an unregistered type using unstructured", func(ctx SpecContext) { By("Creating an object of an unregistered type") item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v7") item.SetKind("Image") item.SetName("my-item") - err := cl.Create(context.Background(), item) + err := cl.Create(ctx, item) Expect(err).ToNot(HaveOccurred()) By("Deleting the object") - err = cl.Delete(context.Background(), item) + err = cl.Delete(ctx, item) Expect(err).ToNot(HaveOccurred()) By("Getting the object") @@ -326,14 +340,41 @@ var _ = Describe("Fake client", func() { item.SetAPIVersion("custom/v7") item.SetKind("Image") item.SetName("my-item") - err = cl.Get(context.Background(), client.ObjectKeyFromObject(item), item) + err = cl.Get(ctx, client.ObjectKeyFromObject(item), item) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should support filtering by labels and their values", func() { + It("should be able to retrieve objects by PartialObjectMetadata", func(ctx SpecContext) { + By("Creating a Resource") + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + } + err := cl.Create(ctx, secret) + Expect(err).ToNot(HaveOccurred()) + + By("Fetching the resource using a PartialObjectMeta") + partialObjMeta := &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "bar", + }, + } + partialObjMeta.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Secret")) + + err = cl.Get(ctx, client.ObjectKeyFromObject(partialObjMeta), partialObjMeta) + Expect(err).ToNot(HaveOccurred()) + + Expect(partialObjMeta.Kind).To(Equal("Secret")) + Expect(partialObjMeta.APIVersion).To(Equal("v1")) + }) + + It("should support filtering by labels and their values", func(ctx SpecContext) { By("Listing deployments with a particular label and value") list := &appsv1.DeploymentList{} - err := cl.List(context.Background(), list, client.InNamespace("ns1"), + err := cl.List(ctx, list, client.InNamespace("ns1"), client.MatchingLabels(map[string]string{ "test-label": "label-value", })) @@ -342,29 +383,25 @@ var _ = Describe("Fake client", func() { Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should support filtering by label existence", func() { + It("should support filtering by label existence", func(ctx SpecContext) { By("Listing deployments with a particular label") list := &appsv1.DeploymentList{} - err := cl.List(context.Background(), list, client.InNamespace("ns1"), + err := cl.List(ctx, list, client.InNamespace("ns1"), client.HasLabels{"test-label"}) Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should be able to Create", func() { + It("should be able to Create", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "new-test-cm", Namespace: "ns2", }, } - err := cl.Create(context.Background(), newcm) + err := cl.Create(ctx, newcm) Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") @@ -373,13 +410,13 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(newcm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1")) }) - It("should error on create with set resourceVersion", func() { + It("should error on create with set resourceVersion", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -388,58 +425,46 @@ var _ = Describe("Fake client", func() { ResourceVersion: "1", }, } - err := cl.Create(context.Background(), newcm) + err := cl.Create(ctx, newcm) Expect(apierrors.IsBadRequest(err)).To(BeTrue()) }) - It("should not change the submitted object if Create failed", func() { + It("should not change the submitted object if Create failed", func(ctx SpecContext) { By("Trying to create an existing configmap") submitted := cm.DeepCopy() submitted.ResourceVersion = "" submittedReference := submitted.DeepCopy() - err := cl.Create(context.Background(), submitted) + err := cl.Create(ctx, submitted) Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) - Expect(submitted).To(Equal(submittedReference)) + Expect(submitted).To(BeComparableTo(submittedReference)) }) - It("should error on Create with empty Name", func() { + It("should error on Create with empty Name", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Namespace: "ns2", }, } - err := cl.Create(context.Background(), newcm) + err := cl.Create(ctx, newcm) Expect(err.Error()).To(Equal("ConfigMap \"\" is invalid: metadata.name: Required value: name is required")) }) - It("should error on Update with empty Name", func() { + It("should error on Update with empty Name", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Namespace: "ns2", }, } - err := cl.Update(context.Background(), newcm) + err := cl.Update(ctx, newcm) Expect(err.Error()).To(Equal("ConfigMap \"\" is invalid: metadata.name: Required value: name is required")) }) - It("should be able to Create with GenerateName", func() { + It("should be able to Create with GenerateName", func(ctx SpecContext) { By("Creating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ GenerateName: "new-test-cm", Namespace: "ns2", @@ -448,12 +473,12 @@ var _ = Describe("Fake client", func() { }, }, } - err := cl.Create(context.Background(), newcm) + err := cl.Create(ctx, newcm) Expect(err).ToNot(HaveOccurred()) By("Listing configmaps with a particular label") list := &corev1.ConfigMapList{} - err = cl.List(context.Background(), list, client.InNamespace("ns2"), + err = cl.List(ctx, list, client.InNamespace("ns2"), client.MatchingLabels(map[string]string{ "test-label": "label-value", })) @@ -462,13 +487,9 @@ var _ = Describe("Fake client", func() { Expect(list.Items[0].Name).NotTo(BeEmpty()) }) - It("should be able to Update", func() { + It("should be able to Update", func(ctx SpecContext) { By("Updating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-cm", Namespace: "ns2", @@ -478,7 +499,7 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm) + err := cl.Update(ctx, newcm) Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") @@ -487,19 +508,15 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(newcm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1000")) }) - It("should allow updates with non-set ResourceVersion for a resource that allows unconditional updates", func() { + It("should allow updates with non-set ResourceVersion for a resource that allows unconditional updates", func(ctx SpecContext) { By("Updating a new configmap") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-cm", Namespace: "ns2", @@ -508,7 +525,7 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm) + err := cl.Update(ctx, newcm) Expect(err).ToNot(HaveOccurred()) By("Getting the configmap") @@ -517,19 +534,42 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(newcm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1000")) }) - It("should reject updates with non-set ResourceVersion for a resource that doesn't allow unconditional updates", func() { + It("should allow patch when the patch sets RV to 'null'", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + original := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "obj", + Namespace: "ns2", + }} + + err := cl.Create(ctx, original) + Expect(err).ToNot(HaveOccurred()) + + newObj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: original.Name, + Namespace: original.Namespace, + Annotations: map[string]string{ + "foo": "bar", + }, + }} + + Expect(cl.Patch(ctx, newObj, client.MergeFrom(original))).To(Succeed()) + + patched := &corev1.ConfigMap{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(original), patched)).To(Succeed()) + Expect(patched.Annotations).To(Equal(map[string]string{"foo": "bar"})) + }) + + It("should reject updates with non-set ResourceVersion for a resource that doesn't allow unconditional updates", func(ctx SpecContext) { By("Creating a new binding") binding := &corev1.Binding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Binding", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-binding", Namespace: "ns2", @@ -541,14 +581,10 @@ var _ = Describe("Fake client", func() { Name: cm.Name, }, } - Expect(cl.Create(context.Background(), binding)).To(Succeed()) + Expect(cl.Create(ctx, binding)).To(Succeed()) By("Updating the binding with a new resource lacking resource version") newBinding := &corev1.Binding{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Binding", - }, ObjectMeta: metav1.ObjectMeta{ Name: binding.Name, Namespace: binding.Namespace, @@ -558,23 +594,19 @@ var _ = Describe("Fake client", func() { Name: "blue", }, } - Expect(cl.Update(context.Background(), newBinding)).NotTo(Succeed()) + Expect(cl.Update(ctx, newBinding)).NotTo(Succeed()) }) - It("should allow create on update for a resource that allows create on update", func() { + It("should allow create on update for a resource that allows create on update", func(ctx SpecContext) { By("Creating a new lease with update") lease := &coordinationv1.Lease{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "coordination.k8s.io/v1", - Kind: "Lease", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-lease", Namespace: "ns2", }, Spec: coordinationv1.LeaseSpec{}, } - Expect(cl.Create(context.Background(), lease)).To(Succeed()) + Expect(cl.Create(ctx, lease)).To(Succeed()) By("Getting the lease") namespacedName := types.NamespacedName{ @@ -582,18 +614,14 @@ var _ = Describe("Fake client", func() { Namespace: lease.Namespace, } obj := &coordinationv1.Lease{} - Expect(cl.Get(context.Background(), namespacedName, obj)).To(Succeed()) + Expect(cl.Get(ctx, namespacedName, obj)).To(Succeed()) Expect(obj).To(Equal(lease)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1")) }) - It("should reject create on update for a resource that does not allow create on update", func() { + It("should reject create on update for a resource that does not allow create on update", func(ctx SpecContext) { By("Attemping to create a new configmap with update") newcm := &corev1.ConfigMap{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "ConfigMap", - }, ObjectMeta: metav1.ObjectMeta{ Name: "different-test-cm", Namespace: "ns2", @@ -602,10 +630,10 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - Expect(cl.Update(context.Background(), newcm)).NotTo(Succeed()) + Expect(cl.Update(ctx, newcm)).NotTo(Succeed()) }) - It("should reject updates with non-matching ResourceVersion", func() { + It("should reject updates with non-matching ResourceVersion", func(ctx SpecContext) { By("Updating a new configmap") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -617,7 +645,7 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm) + err := cl.Update(ctx, newcm) Expect(apierrors.IsConflict(err)).To(BeTrue()) By("Getting the configmap") @@ -626,67 +654,109 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) + Expect(err).ToNot(HaveOccurred()) + Expect(obj).To(Equal(cm)) + Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) + }) + + It("should reject apply with non-matching ResourceVersion", func(ctx SpecContext) { + cl := NewClientBuilder().WithRuntimeObjects(cm).Build() + applyCM := corev1applyconfigurations.ConfigMap(cm.Name, cm.Namespace).WithResourceVersion("0") + err := cl.Apply(ctx, applyCM, client.FieldOwner("test")) + Expect(apierrors.IsConflict(err)).To(BeTrue()) + + obj := &corev1.ConfigMap{} + err = cl.Get(ctx, client.ObjectKeyFromObject(cm), obj) Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(cm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) }) - It("should reject Delete with a mismatched ResourceVersion", func() { + It("should reject Delete with a mismatched ResourceVersion", func(ctx SpecContext) { bogusRV := "bogus" By("Deleting with a mismatched ResourceVersion Precondition") - err := cl.Delete(context.Background(), dep, client.Preconditions{ResourceVersion: &bogusRV}) + err := cl.Delete(ctx, dep, client.Preconditions{ResourceVersion: &bogusRV}) Expect(apierrors.IsConflict(err)).To(BeTrue()) list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) + err = cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(2)) Expect(list.Items).To(ConsistOf(*dep, *dep2)) }) - It("should successfully Delete with a matching ResourceVersion", func() { + It("should successfully Delete with a matching ResourceVersion", func(ctx SpecContext) { goodRV := trackerAddResourceVersion By("Deleting with a matching ResourceVersion Precondition") - err := cl.Delete(context.Background(), dep, client.Preconditions{ResourceVersion: &goodRV}) + err := cl.Delete(ctx, dep, client.Preconditions{ResourceVersion: &goodRV}) Expect(err).ToNot(HaveOccurred()) list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) + err = cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should be able to Delete with no ResourceVersion Precondition", func() { + It("should be able to Delete with no ResourceVersion Precondition", func(ctx SpecContext) { By("Deleting a deployment") - err := cl.Delete(context.Background(), dep) + err := cl.Delete(ctx, dep) Expect(err).ToNot(HaveOccurred()) By("Listing all deployments in the namespace") list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) + err = cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should be able to Delete with no opts even if object's ResourceVersion doesn't match server", func() { + It("should be able to Delete with no opts even if object's ResourceVersion doesn't match server", func(ctx SpecContext) { By("Deleting a deployment") depCopy := dep.DeepCopy() depCopy.ResourceVersion = "bogus" - err := cl.Delete(context.Background(), depCopy) + err := cl.Delete(ctx, depCopy) Expect(err).ToNot(HaveOccurred()) By("Listing all deployments in the namespace") list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) + err = cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(HaveLen(1)) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("should handle finalizers on Update", func() { + It("should handle finalizers in Apply ", func(ctx SpecContext) { + cl := client.WithFieldOwner(cl, "test") + + By("Creating the object with a finalizer") + cm := corev1applyconfigurations.ConfigMap("test-cm", "delete-with-finalizers"). + WithFinalizers("finalizers.sigs.k8s.io/test") + Expect(cl.Apply(ctx, cm)).To(Succeed()) + + By("Deleting the object") + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: *cm.Name, + Namespace: *cm.Namespace, + }} + Expect(cl.Delete(ctx, obj)).NotTo(HaveOccurred()) + + By("Getting the object") + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).NotTo(HaveOccurred()) + Expect(obj.DeletionTimestamp).NotTo(BeNil()) + + By("Removing the finalizer through SSA") + cm.ResourceVersion = nil + cm.Finalizers = nil + Expect(cl.Apply(ctx, cm)).NotTo(HaveOccurred()) + + By("Getting the object") + err := cl.Get(ctx, client.ObjectKeyFromObject(obj), &corev1.ConfigMap{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should handle finalizers on Update", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "delete-with-finalizers", @@ -702,31 +772,31 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) + err := cl.Create(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Deleting the object") - err = cl.Delete(context.Background(), newObj) + err = cl.Delete(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj.DeletionTimestamp).NotTo(BeNil()) By("Removing the finalizer") obj.Finalizers = []string{} - err = cl.Update(context.Background(), obj) + err = cl.Update(ctx, obj) Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj = &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should reject changes to deletionTimestamp on Update", func() { + It("should reject changes to deletionTimestamp on Update", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "reject-with-deletiontimestamp", @@ -741,52 +811,52 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) + err := cl.Create(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj.DeletionTimestamp).To(BeNil()) By("Adding deletionTimestamp") now := metav1.Now() obj.DeletionTimestamp = &now - err = cl.Update(context.Background(), obj) + err = cl.Update(ctx, obj) Expect(err).To(HaveOccurred()) By("Deleting the object") - err = cl.Delete(context.Background(), newObj) + err = cl.Delete(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Changing the deletionTimestamp to new value") obj = &corev1.ConfigMap{} t := metav1.NewTime(time.Now().Add(time.Second)) obj.DeletionTimestamp = &t - err = cl.Update(context.Background(), obj) + err = cl.Update(ctx, obj) Expect(err).To(HaveOccurred()) By("Removing deletionTimestamp") obj.DeletionTimestamp = nil - err = cl.Update(context.Background(), obj) + err = cl.Update(ctx, obj) Expect(err).To(HaveOccurred()) }) - It("should be able to Delete a Collection", func() { + It("should be able to Delete a Collection", func(ctx SpecContext) { By("Deleting a deploymentList") - err := cl.DeleteAllOf(context.Background(), &appsv1.Deployment{}, client.InNamespace("ns1")) + err := cl.DeleteAllOf(ctx, &appsv1.Deployment{}, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) By("Listing all deployments in the namespace") list := &appsv1.DeploymentList{} - err = cl.List(context.Background(), list, client.InNamespace("ns1")) + err = cl.List(ctx, list, client.InNamespace("ns1")) Expect(err).ToNot(HaveOccurred()) Expect(list.Items).To(BeEmpty()) }) - It("should handle finalizers deleting a collection", func() { + It("should handle finalizers deleting a collection", func(ctx SpecContext) { for i := 0; i < 5; i++ { namespacedName := types.NamespacedName{ Name: fmt.Sprintf("test-cm-%d", i), @@ -803,16 +873,16 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) + err := cl.Create(ctx, newObj) Expect(err).ToNot(HaveOccurred()) } By("Deleting the object") - err := cl.DeleteAllOf(context.Background(), &corev1.ConfigMap{}, client.InNamespace("delete-collection-with-finalizers")) + err := cl.DeleteAllOf(ctx, &corev1.ConfigMap{}, client.InNamespace("delete-collection-with-finalizers")) Expect(err).ToNot(HaveOccurred()) configmaps := corev1.ConfigMapList{} - err = cl.List(context.Background(), &configmaps, client.InNamespace("delete-collection-with-finalizers")) + err = cl.List(ctx, &configmaps, client.InNamespace("delete-collection-with-finalizers")) Expect(err).ToNot(HaveOccurred()) Expect(configmaps.Items).To(HaveLen(5)) @@ -821,9 +891,9 @@ var _ = Describe("Fake client", func() { } }) - It("should be able to watch", func() { + It("should be able to watch", func(ctx SpecContext) { By("Creating a watch") - objWatch, err := cl.Watch(context.Background(), &corev1.ServiceList{}) + objWatch, err := cl.Watch(ctx, &corev1.ServiceList{}) Expect(err).NotTo(HaveOccurred()) defer objWatch.Stop() @@ -834,7 +904,7 @@ var _ = Describe("Fake client", func() { // in the outer routine, sleep to make sure this is always true time.Sleep(100 * time.Millisecond) - err := cl.Create(context.Background(), &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "for-watch"}}) + err := cl.Create(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "for-watch"}}) Expect(err).ToNot(HaveOccurred()) }() @@ -848,7 +918,7 @@ var _ = Describe("Fake client", func() { }) Context("with the DryRun option", func() { - It("should not create a new object", func() { + It("should not create a new object", func(ctx SpecContext) { By("Creating a new configmap with DryRun") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -856,7 +926,7 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", }, } - err := cl.Create(context.Background(), newcm, client.DryRunAll) + err := cl.Create(ctx, newcm, client.DryRunAll) Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") @@ -865,13 +935,13 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) Expect(obj).NotTo(Equal(newcm)) }) - It("should not Update the object", func() { + It("should not Update the object", func(ctx SpecContext) { By("Updating a new configmap with DryRun") newcm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -883,7 +953,7 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Update(context.Background(), newcm, client.DryRunAll) + err := cl.Update(ctx, newcm, client.DryRunAll) Expect(err).ToNot(HaveOccurred()) By("Getting the new configmap") @@ -892,19 +962,19 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(cm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) }) - It("Should not Delete the object", func() { + It("Should not Delete the object", func(ctx SpecContext) { By("Deleting a configmap with DryRun with Delete()") - err := cl.Delete(context.Background(), cm, client.DryRunAll) + err := cl.Delete(ctx, cm, client.DryRunAll) Expect(err).ToNot(HaveOccurred()) By("Deleting a configmap with DryRun with DeleteAllOf()") - err = cl.DeleteAllOf(context.Background(), cm, client.DryRunAll) + err = cl.DeleteAllOf(ctx, cm, client.DryRunAll) Expect(err).ToNot(HaveOccurred()) By("Getting the configmap") @@ -913,14 +983,14 @@ var _ = Describe("Fake client", func() { Namespace: "ns2", } obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj).To(Equal(cm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) }) }) - It("should be able to Patch", func() { + It("should be able to Patch", func(ctx SpecContext) { By("Patching a deployment") mergePatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ @@ -930,7 +1000,7 @@ var _ = Describe("Fake client", func() { }, }) Expect(err).NotTo(HaveOccurred()) - err = cl.Patch(context.Background(), dep, client.RawPatch(types.StrategicMergePatchType, mergePatch)) + err = cl.Patch(ctx, dep, client.RawPatch(types.StrategicMergePatchType, mergePatch)) Expect(err).NotTo(HaveOccurred()) By("Getting the patched deployment") @@ -939,13 +1009,13 @@ var _ = Describe("Fake client", func() { Namespace: "ns1", } obj := &appsv1.Deployment{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).NotTo(HaveOccurred()) Expect(obj.Annotations["foo"]).To(Equal("bar")) Expect(obj.ObjectMeta.ResourceVersion).To(Equal("1000")) }) - It("should ignore deletionTimestamp without finalizer on Create", func() { + It("should ignore deletionTimestamp without finalizer on Create", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "ignore-deletiontimestamp", @@ -964,18 +1034,18 @@ var _ = Describe("Fake client", func() { }, } - err := cl.Create(context.Background(), newObj) + err := cl.Create(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj.DeletionTimestamp).To(BeNil()) }) - It("should reject deletionTimestamp without finalizers on Build", func() { + It("should reject deletionTimestamp without finalizers on Build", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "reject-deletiontimestamp-no-finalizers", @@ -1012,12 +1082,12 @@ var _ = Describe("Fake client", func() { By("Getting the object") obj = &corev1.ConfigMap{} - err := cl.Get(context.Background(), namespacedName, obj) + err := cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) }) - It("should reject changes to deletionTimestamp on Patch", func() { + It("should reject changes to deletionTimestamp on Patch", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "reject-deletiontimestamp", @@ -1034,7 +1104,7 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) + err := cl.Create(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Add a deletionTimestamp") @@ -1046,16 +1116,16 @@ var _ = Describe("Fake client", func() { DeletionTimestamp: &now, }, } - err = cl.Patch(context.Background(), obj, client.MergeFrom(newObj)) + err = cl.Patch(ctx, obj, client.MergeFrom(newObj)) Expect(err).To(HaveOccurred()) By("Deleting the object") - err = cl.Delete(context.Background(), newObj) + err = cl.Delete(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj = &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) Expect(obj.DeletionTimestamp).NotTo(BeNil()) @@ -1063,7 +1133,7 @@ var _ = Describe("Fake client", func() { newObj = &corev1.ConfigMap{} t := metav1.NewTime(time.Now().Add(time.Second)) newObj.DeletionTimestamp = &t - err = cl.Patch(context.Background(), newObj, client.MergeFrom(obj)) + err = cl.Patch(ctx, newObj, client.MergeFrom(obj)) Expect(err).To(HaveOccurred()) By("Removing deletionTimestamp") @@ -1074,12 +1144,12 @@ var _ = Describe("Fake client", func() { DeletionTimestamp: nil, }, } - err = cl.Patch(context.Background(), newObj, client.MergeFrom(obj)) + err = cl.Patch(ctx, newObj, client.MergeFrom(obj)) Expect(err).To(HaveOccurred()) }) - It("should handle finalizers on Patch", func() { + It("should handle finalizers on Patch", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "delete-with-finalizers", @@ -1095,11 +1165,11 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), newObj) + err := cl.Create(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Deleting the object") - err = cl.Delete(context.Background(), newObj) + err = cl.Delete(ctx, newObj) Expect(err).ToNot(HaveOccurred()) By("Removing the finalizer") @@ -1110,16 +1180,16 @@ var _ = Describe("Fake client", func() { Finalizers: []string{}, }, } - err = cl.Patch(context.Background(), obj, client.MergeFrom(newObj)) + err = cl.Patch(ctx, obj, client.MergeFrom(newObj)) Expect(err).ToNot(HaveOccurred()) By("Getting the object") obj = &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, obj) + err = cl.Get(ctx, namespacedName, obj) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should remove finalizers of the object on Patch", func() { + It("should remove finalizers of the object on Patch", func(ctx SpecContext) { namespacedName := types.NamespacedName{ Name: "test-cm", Namespace: "patch-finalizers-in-obj", @@ -1135,7 +1205,7 @@ var _ = Describe("Fake client", func() { "test-key": "new-value", }, } - err := cl.Create(context.Background(), obj) + err := cl.Create(ctx, obj) Expect(err).ToNot(HaveOccurred()) By("Removing the finalizer") @@ -1147,7 +1217,7 @@ var _ = Describe("Fake client", func() { }, }) Expect(err).ToNot(HaveOccurred()) - err = cl.Patch(context.Background(), obj, client.RawPatch(types.StrategicMergePatchType, mergePatch)) + err = cl.Patch(ctx, obj, client.RawPatch(types.StrategicMergePatchType, mergePatch)) Expect(err).ToNot(HaveOccurred()) By("Check the finalizer has been removed in the object") @@ -1155,7 +1225,7 @@ var _ = Describe("Fake client", func() { By("Check the finalizer has been removed in client") newObj := &corev1.ConfigMap{} - err = cl.Get(context.Background(), namespacedName, newObj) + err = cl.Get(ctx, namespacedName, newObj) Expect(err).ToNot(HaveOccurred()) Expect(newObj.Finalizers).To(BeEmpty()) }) @@ -1224,79 +1294,140 @@ var _ = Describe("Fake client", func() { }) Context("filtered List using field selector", func() { - It("errors when there's no Index for the GroupVersionResource", func() { + It("errors when there's no Index for the GroupVersionResource", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("key", "val"), + } + list := &corev1.ConfigMapList{} + err := cl.List(ctx, list, listOpts) + Expect(err).To(HaveOccurred()) + Expect(list.Items).To(BeEmpty()) + }) + + It("errors when there's no Index for the GroupVersionResource with UnstructuredList", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("key", "val"), } - err := cl.List(context.Background(), &corev1.ConfigMapList{}, listOpts) + list := &unstructured.UnstructuredList{} + list.SetAPIVersion("v1") + list.SetKind("ConfigMapList") + err := cl.List(ctx, list, listOpts) Expect(err).To(HaveOccurred()) + Expect(list.GroupVersionKind().GroupVersion().String()).To(Equal("v1")) + Expect(list.GetKind()).To(Equal("ConfigMapList")) + Expect(list.Items).To(BeEmpty()) }) - It("errors when there's no Index matching the field name", func() { + It("errors when there's no Index matching the field name", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.paused", "false"), } - err := cl.List(context.Background(), &appsv1.DeploymentList{}, listOpts) + list := &appsv1.DeploymentList{} + err := cl.List(ctx, list, listOpts) Expect(err).To(HaveOccurred()) + Expect(list.Items).To(BeEmpty()) }) - It("errors when field selector uses two requirements", func() { + It("errors when field selector uses two requirements", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.AndSelectors( fields.OneTermEqualSelector("spec.replicas", "1"), fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), )} - err := cl.List(context.Background(), &appsv1.DeploymentList{}, listOpts) + list := &appsv1.DeploymentList{} + err := cl.List(ctx, list, listOpts) Expect(err).To(HaveOccurred()) + Expect(list.Items).To(BeEmpty()) }) - It("returns two deployments that match the only field selector requirement", func() { + It("returns two deployments that match the only field selector requirement", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), } list := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) Expect(list.Items).To(ConsistOf(*dep, *dep2)) }) - It("returns no object because no object matches the only field selector requirement", func() { + It("returns no object because no object matches the only field selector requirement", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), } list := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) Expect(list.Items).To(BeEmpty()) }) - It("returns deployment that matches both the field and label selectors", func() { + It("returns deployment that matches both the field and label selectors", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), LabelSelector: labels.SelectorFromSet(dep2.Labels), } list := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) Expect(list.Items).To(ConsistOf(*dep2)) }) - It("returns no object even if field selector matches because label selector doesn't", func() { + It("returns no object even if field selector matches because label selector doesn't", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), LabelSelector: labels.Nothing(), } list := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) Expect(list.Items).To(BeEmpty()) }) - It("returns no object even if label selector matches because field selector doesn't", func() { + It("returns no object even if label selector matches because field selector doesn't", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), LabelSelector: labels.Everything(), } list := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) Expect(list.Items).To(BeEmpty()) }) + + It("supports adding an index at runtime", func(ctx SpecContext) { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", "test-deployment-2"), + } + list := &appsv1.DeploymentList{} + err := cl.List(ctx, list, listOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no index with name metadata.name has been registered")) + + err = AddIndex(cl, &appsv1.Deployment{}, "metadata.name", func(obj client.Object) []string { + return []string{obj.GetName()} + }) + Expect(err).To(Succeed()) + + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep2)) + }) + + It("Is not a datarace to add and use indexes in parallel", func(ctx SpecContext) { + wg := sync.WaitGroup{} + wg.Add(2) + + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), + } + go func() { + defer wg.Done() + defer GinkgoRecover() + Expect(cl.List(ctx, &appsv1.DeploymentList{}, listOpts)).To(Succeed()) + }() + go func() { + defer wg.Done() + defer GinkgoRecover() + err := AddIndex(cl, &appsv1.Deployment{}, "metadata.name", func(obj client.Object) []string { + return []string{obj.GetName()} + }) + Expect(err).To(Succeed()) + }() + wg.Wait() + }) }) }) @@ -1310,48 +1441,45 @@ var _ = Describe("Fake client", func() { }) Context("filtered List using field selector", func() { - It("uses the second index to retrieve the indexed objects when there are matches", func() { + It("uses the second index to retrieve the indexed objects when there are matches", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), } list := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) Expect(list.Items).To(ConsistOf(*dep)) }) - It("uses the second index to retrieve the indexed objects when there are no matches", func() { + It("uses the second index to retrieve the indexed objects when there are no matches", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RollingUpdateDeploymentStrategyType)), } list := &appsv1.DeploymentList{} - Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) Expect(list.Items).To(BeEmpty()) }) - It("errors when field selector uses two requirements", func() { + It("no error when field selector uses two requirements", func(ctx SpecContext) { listOpts := &client.ListOptions{ FieldSelector: fields.AndSelectors( fields.OneTermEqualSelector("spec.replicas", "1"), fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), )} - err := cl.List(context.Background(), &appsv1.DeploymentList{}, listOpts) - Expect(err).To(HaveOccurred()) + list := &appsv1.DeploymentList{} + Expect(cl.List(ctx, list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep)) }) }) }) }) - It("should set the ResourceVersion to 999 when adding an object to the tracker", func() { + It("should set the ResourceVersion to 999 when adding an object to the tracker", func(ctx SpecContext) { cl := NewClientBuilder().WithObjects(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}).Build() retrieved := &corev1.Secret{} - Expect(cl.Get(context.Background(), types.NamespacedName{Name: "cm"}, retrieved)).To(Succeed()) + Expect(cl.Get(ctx, types.NamespacedName{Name: "cm"}, retrieved)).To(Succeed()) reference := &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, ObjectMeta: metav1.ObjectMeta{ Name: "cm", ResourceVersion: "999", @@ -1360,7 +1488,7 @@ var _ = Describe("Fake client", func() { Expect(retrieved).To(Equal(reference)) }) - It("should be able to build with given tracker and get resource", func() { + It("should be able to build with given tracker and get resource", func(ctx SpecContext) { clientSet := fake.NewSimpleClientset(dep) cl := NewClientBuilder().WithRuntimeObjects(dep2).WithObjectTracker(clientSet.Tracker()).Build() @@ -1370,12 +1498,12 @@ var _ = Describe("Fake client", func() { Namespace: "ns1", } obj := &appsv1.Deployment{} - err := cl.Get(context.Background(), namespacedName, obj) + err := cl.Get(ctx, namespacedName, obj) Expect(err).ToNot(HaveOccurred()) - Expect(obj).To(Equal(dep)) + Expect(obj).To(BeComparableTo(dep)) By("Getting a deployment from clientSet") - csDep2, err := clientSet.AppsV1().Deployments("ns1").Get(context.Background(), "test-deployment-2", metav1.GetOptions{}) + csDep2, err := clientSet.AppsV1().Deployments("ns1").Get(ctx, "test-deployment-2", metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(csDep2).To(Equal(dep2)) @@ -1386,10 +1514,6 @@ var _ = Describe("Fake client", func() { } dep3 := &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "Deployment", - }, ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment-3", Namespace: "ns1", @@ -1400,35 +1524,32 @@ var _ = Describe("Fake client", func() { }, } - _, err = clientSet.AppsV1().Deployments("ns1").Create(context.Background(), dep3, metav1.CreateOptions{}) + _, err = clientSet.AppsV1().Deployments("ns1").Create(ctx, dep3, metav1.CreateOptions{}) Expect(err).ToNot(HaveOccurred()) obj = &appsv1.Deployment{} - err = cl.Get(context.Background(), namespacedName3, obj) + err = cl.Get(ctx, namespacedName3, obj) Expect(err).ToNot(HaveOccurred()) - Expect(obj).To(Equal(dep3)) + Expect(obj).To(BeComparableTo(dep3)) }) - It("should not change the status of typed objects that have a status subresource on update", func() { - obj := &corev1.Node{ + It("should not change the status of typed objects that have a status subresource on update", func(ctx SpecContext) { + obj := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "node", - }, - Status: corev1.NodeStatus{ - NodeInfo: corev1.NodeSystemInfo{MachineID: "machine-id"}, + Name: "pod", }, } cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() - obj.Status.NodeInfo.MachineID = "updated-machine-id" - Expect(cl.Update(context.Background(), obj)).To(Succeed()) + obj.Status.Phase = "Running" + Expect(cl.Update(ctx, obj)).To(Succeed()) - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) - Expect(obj.Status).To(BeEquivalentTo(corev1.NodeStatus{NodeInfo: corev1.NodeSystemInfo{MachineID: "machine-id"}})) + Expect(obj.Status).To(BeEquivalentTo(corev1.PodStatus{})) }) - It("should return a conflict error when an incorrect RV is used on status update", func() { + It("should return a conflict error when an incorrect RV is used on status update", func(ctx SpecContext) { obj := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node", @@ -1439,11 +1560,11 @@ var _ = Describe("Fake client", func() { obj.Status.Phase = corev1.NodeRunning obj.ResourceVersion = "invalid" - err := cl.Status().Update(context.Background(), obj) + err := cl.Status().Update(ctx, obj) Expect(apierrors.IsConflict(err)).To(BeTrue()) }) - It("should not change non-status field of typed objects that have a status subresource on status update", func() { + It("should not change non-status field of typed objects that have a status subresource on status update", func(ctx SpecContext) { obj := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node", @@ -1460,43 +1581,105 @@ var _ = Describe("Fake client", func() { cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() objOriginal := obj.DeepCopy() - obj.Spec.PodCIDR = "cidr-from-status-update" - obj.Status.NodeInfo.MachineID = "machine-id-from-status-update" - Expect(cl.Status().Update(context.Background(), obj)).NotTo(HaveOccurred()) + obj.Spec.PodCIDR = cidrFromStatusUpdate + obj.Annotations = map[string]string{ + "some-annotation-key": "some-annotation-value", + } + obj.Labels = map[string]string{ + "some-label-key": "some-label-value", + } + + obj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) objOriginal.APIVersion = actual.APIVersion objOriginal.Kind = actual.Kind objOriginal.ResourceVersion = actual.ResourceVersion - objOriginal.Status.NodeInfo.MachineID = "machine-id-from-status-update" + objOriginal.Status.NodeInfo.MachineID = machineIDFromStatusUpdate Expect(cmp.Diff(objOriginal, actual)).To(BeEmpty()) }) - It("should not change the status of typed objects that have a status subresource on patch", func() { + It("should be able to update an object after updating an object's status", func(ctx SpecContext) { obj := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node", }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, Status: corev1.NodeStatus{ NodeInfo: corev1.NodeSystemInfo{ MachineID: "machine-id", }, }, } - Expect(cl.Create(context.Background(), obj)).To(Succeed()) - original := obj.DeepCopy() + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + expectedObj := obj.DeepCopy() + + obj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) + + obj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + expectedObj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + Expect(cl.Update(ctx, obj)).NotTo(HaveOccurred()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + + expectedObj.APIVersion = actual.APIVersion + expectedObj.Kind = actual.Kind + expectedObj.ResourceVersion = actual.ResourceVersion + expectedObj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cmp.Diff(expectedObj, actual)).To(BeEmpty()) + }) + + It("should be able to update an object's status after updating an object", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + MachineID: "machine-id", + }, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + expectedObj := obj.DeepCopy() - obj.Status.NodeInfo.MachineID = "machine-id-from-patch" - Expect(cl.Patch(context.Background(), obj, client.MergeFrom(original))).To(Succeed()) + obj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + expectedObj.Annotations = map[string]string{ + "some-annotation-key": "some", + } + Expect(cl.Update(ctx, obj)).NotTo(HaveOccurred()) - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + obj.Spec.PodCIDR = cidrFromStatusUpdate + obj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) - Expect(obj.Status).To(BeEquivalentTo(corev1.NodeStatus{NodeInfo: corev1.NodeSystemInfo{MachineID: "machine-id"}})) + expectedObj.APIVersion = actual.APIVersion + expectedObj.Kind = actual.Kind + expectedObj.ResourceVersion = actual.ResourceVersion + expectedObj.Status.NodeInfo.MachineID = machineIDFromStatusUpdate + Expect(cmp.Diff(expectedObj, actual)).To(BeEmpty()) }) - It("should not change non-status field of typed objects that have a status subresource on status patch", func() { + It("Should only override status fields of typed objects that have a status subresource on status update", func(ctx SpecContext) { obj := &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node", @@ -1504,89 +1687,342 @@ var _ = Describe("Fake client", func() { Spec: corev1.NodeSpec{ PodCIDR: "old-cidr", }, + Status: corev1.NodeStatus{ + NodeInfo: corev1.NodeSystemInfo{ + MachineID: "machine-id", + }, + }, } cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() objOriginal := obj.DeepCopy() - obj.Spec.PodCIDR = "cidr-from-status-update" - obj.Status.NodeInfo.MachineID = "machine-id" - Expect(cl.Status().Patch(context.Background(), obj, client.MergeFrom(objOriginal))).NotTo(HaveOccurred()) + obj.Status.Phase = corev1.NodeRunning + Expect(cl.Status().Update(ctx, obj)).NotTo(HaveOccurred()) actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) objOriginal.APIVersion = actual.APIVersion objOriginal.Kind = actual.Kind objOriginal.ResourceVersion = actual.ResourceVersion - objOriginal.Status.NodeInfo.MachineID = "machine-id" - Expect(cmp.Diff(objOriginal, actual)).To(BeEmpty()) + Expect(cmp.Diff(objOriginal, actual)).ToNot(BeEmpty()) + Expect(objOriginal.Status.NodeInfo.MachineID).To(Equal(actual.Status.NodeInfo.MachineID)) + Expect(objOriginal.Status.Phase).ToNot(Equal(actual.Status.Phase)) }) - It("should not change the status of unstructured objects that are configured to have a status subresource on update", func() { - obj := &unstructured.Unstructured{} - obj.SetAPIVersion("foo/v1") - obj.SetKind("Foo") - obj.SetName("a-foo") - - err := unstructured.SetNestedField(obj.Object, map[string]any{"state": "old"}, "status") - Expect(err).NotTo(HaveOccurred()) + It("should be able to change typed objects that have a scale subresource on patch", func(ctx SpecContext) { + obj := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deploy", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + objOriginal := obj.DeepCopy() - cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + patch := []byte(fmt.Sprintf(`{"spec":{"replicas":%d}}`, 2)) + Expect(cl.SubResource("scale").Patch(ctx, obj, client.RawPatch(types.MergePatchType, patch))).NotTo(HaveOccurred()) - err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") - Expect(err).ToNot(HaveOccurred()) + actual := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).To(Succeed()) - Expect(cl.Update(context.Background(), obj)).To(Succeed()) + objOriginal.APIVersion = actual.APIVersion + objOriginal.Kind = actual.Kind + objOriginal.ResourceVersion = actual.ResourceVersion + objOriginal.Spec.Replicas = ptr.To(int32(2)) + Expect(cmp.Diff(objOriginal, actual)).To(BeEmpty()) + }) - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + It("should not change the status of typed objects that have a status subresource on patch", func(ctx SpecContext) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + } + Expect(cl.Create(ctx, obj)).To(Succeed()) + original := obj.DeepCopy() - Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "old"})) - }) + obj.Status.Phase = "Running" + Expect(cl.Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) - It("should not change non-status fields of unstructured objects that are configured to have a status subresource on status update", func() { - obj := &unstructured.Unstructured{} - obj.SetAPIVersion("foo/v1") - obj.SetKind("Foo") - obj.SetName("a-foo") + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) - err := unstructured.SetNestedField(obj.Object, "original", "spec") - Expect(err).NotTo(HaveOccurred()) + Expect(obj.Status).To(BeEquivalentTo(corev1.PodStatus{})) + }) + It("should not change non-status field of typed objects that have a status subresource on status patch", func(ctx SpecContext) { + obj := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + } cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + objOriginal := obj.DeepCopy() - err = unstructured.SetNestedField(obj.Object, "from-status-update", "spec") - Expect(err).NotTo(HaveOccurred()) - err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") - Expect(err).ToNot(HaveOccurred()) + obj.Spec.PodCIDR = cidrFromStatusUpdate + obj.Status.NodeInfo.MachineID = "machine-id" + Expect(cl.Status().Patch(ctx, obj, client.MergeFrom(objOriginal))).NotTo(HaveOccurred()) - Expect(cl.Status().Update(context.Background(), obj)).To(Succeed()) - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).NotTo(HaveOccurred()) - Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "new"})) - Expect(obj.Object["spec"]).To(BeEquivalentTo("original")) + objOriginal.APIVersion = actual.APIVersion + objOriginal.Kind = actual.Kind + objOriginal.ResourceVersion = actual.ResourceVersion + objOriginal.Status.NodeInfo.MachineID = "machine-id" + Expect(cmp.Diff(objOriginal, actual)).To(BeEmpty()) }) - It("should not change the status of unstructured objects that are configured to have a status subresource on patch", func() { - obj := &unstructured.Unstructured{} - obj.SetAPIVersion("foo/v1") - obj.SetKind("Foo") - obj.SetName("a-foo") - cl := NewClientBuilder().WithStatusSubresource(obj).Build() + It("should not change the status of objects with status subresource when creating through apply ", func(ctx SpecContext) { + obj := corev1applyconfigurations. + Pod("node", ""). + WithStatus( + corev1applyconfigurations.PodStatus().WithPhase("Running"), + ) - Expect(cl.Create(context.Background(), obj)).To(Succeed()) - original := obj.DeepCopy() + cl := NewClientBuilder().WithStatusSubresource(&corev1.Pod{}).Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("test"))).To(Succeed()) - err := unstructured.SetNestedField(obj.Object, map[string]interface{}{"count": int64(2)}, "status") - Expect(err).ToNot(HaveOccurred()) - Expect(cl.Patch(context.Background(), obj, client.MergeFrom(original))).To(Succeed()) + p := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: *obj.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(p), p)).To(Succeed()) - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + Expect(p.Status).To(BeComparableTo(corev1.PodStatus{})) + }) - Expect(obj.Object["status"]).To(BeNil()) + It("should not change the status of objects with status subresource when updating through apply ", func(ctx SpecContext) { - }) + cl := NewClientBuilder().WithStatusSubresource(&corev1.Pod{}).Build() + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod"}} + Expect(cl.Create(ctx, pod)).NotTo(HaveOccurred()) - It("should not change non-status fields of unstructured objects that are configured to have a status subresource on status patch", func() { + obj := corev1applyconfigurations. + Pod(pod.Name, ""). + WithStatus( + corev1applyconfigurations.PodStatus().WithPhase("Running"), + ) + Expect(cl.Apply(ctx, obj, client.FieldOwner("test"))).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + + Expect(pod.Status).To(BeComparableTo(corev1.PodStatus{})) + }) + + It("should only change status on status apply", func(ctx SpecContext) { + initial := &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + }, + Spec: corev1.NodeSpec{ + PodCIDR: "old-cidr", + }, + } + cl := NewClientBuilder().WithStatusSubresource(&corev1.Node{}).WithObjects(initial).Build() + + ac := corev1applyconfigurations.Node(initial.Name). + WithSpec(corev1applyconfigurations.NodeSpec().WithPodCIDR(initial.Spec.PodCIDR + "-updated")). + WithStatus(corev1applyconfigurations.NodeStatus().WithPhase(corev1.NodeRunning)) + + Expect(cl.Status().Apply(ctx, ac, client.FieldOwner("test-owner"))).To(Succeed()) + + actual := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: initial.Name}} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(actual), actual)).To(Succeed()) + + initial.ResourceVersion = actual.ResourceVersion + initial.Status = actual.Status + Expect(initial).To(BeComparableTo(actual)) + }) + + It("should Unmarshal the schemaless object with int64 to preserve ints", func(ctx SpecContext) { + schemeBuilder := &scheme.Builder{GroupVersion: schema.GroupVersion{Group: "test", Version: "v1"}} + schemeBuilder.Register(&WithSchemalessSpec{}) + + scheme := runtime.NewScheme() + Expect(schemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) + + spec := Schemaless{ + "key": int64(1), + } + + obj := &WithSchemalessSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-foo", + }, + Spec: spec, + } + cl := NewClientBuilder().WithScheme(scheme).WithStatusSubresource(obj).WithObjects(obj).Build() + + Expect(cl.Update(ctx, obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + }) + + It("should Unmarshal the schemaless object with float64 to preserve ints", func(ctx SpecContext) { + schemeBuilder := &scheme.Builder{GroupVersion: schema.GroupVersion{Group: "test", Version: "v1"}} + schemeBuilder.Register(&WithSchemalessSpec{}) + + scheme := runtime.NewScheme() + Expect(schemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) + + spec := Schemaless{ + "key": 1.1, + } + + obj := &WithSchemalessSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "a-foo", + }, + Spec: spec, + } + cl := NewClientBuilder().WithScheme(scheme).WithStatusSubresource(obj).WithObjects(obj).Build() + + Expect(cl.Update(ctx, obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + Expect(obj.Spec).To(BeEquivalentTo(spec)) + }) + + It("should not change the status of unstructured objects that are configured to have a status subresource on update", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + + err := unstructured.SetNestedField(obj.Object, map[string]any{"state": "old"}, "status") + Expect(err).NotTo(HaveOccurred()) + + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") + Expect(err).ToNot(HaveOccurred()) + + Expect(cl.Update(ctx, obj)).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "old"})) + }) + + It("should not change non-status fields of unstructured objects that are configured to have a status subresource on status update", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + + err := unstructured.SetNestedField(obj.Object, "original", "spec") + Expect(err).NotTo(HaveOccurred()) + + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + err = unstructured.SetNestedField(obj.Object, "from-status-update", "spec") + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") + Expect(err).ToNot(HaveOccurred()) + + Expect(cl.Status().Update(ctx, obj)).To(Succeed()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "new"})) + Expect(obj.Object["spec"]).To(BeEquivalentTo("original")) + }) + + It("should not change the status of known unstructured objects that have a status subresource on update", func(ctx SpecContext) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + // update using unstructured + u := &unstructured.Unstructured{} + u.SetAPIVersion("v1") + u.SetKind("Pod") + u.SetName(obj.Name) + err := cl.Get(ctx, client.ObjectKeyFromObject(u), u) + Expect(err).NotTo(HaveOccurred()) + + err = unstructured.SetNestedField(u.Object, string(corev1.RestartPolicyNever), "spec", "restartPolicy") + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(u.Object, string(corev1.PodRunning), "status", "phase") + Expect(err).NotTo(HaveOccurred()) + + Expect(cl.Update(ctx, u)).To(Succeed()) + + actual := &corev1.Pod{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), actual)).To(Succeed()) + obj.ResourceVersion = actual.ResourceVersion + // only the spec mutation should persist + obj.Spec.RestartPolicy = corev1.RestartPolicyNever + Expect(cmp.Diff(obj, actual)).To(BeEmpty()) + }) + + It("should not change non-status field of known unstructured objects that have a status subresource on status update", func(ctx SpecContext) { + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + cl := NewClientBuilder().WithStatusSubresource(obj).WithObjects(obj).Build() + + // status update using unstructured + u := &unstructured.Unstructured{} + u.SetAPIVersion("v1") + u.SetKind("Pod") + u.SetName(obj.Name) + err := cl.Get(ctx, client.ObjectKeyFromObject(u), u) + Expect(err).NotTo(HaveOccurred()) + + err = unstructured.SetNestedField(u.Object, string(corev1.RestartPolicyNever), "spec", "restartPolicy") + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(u.Object, string(corev1.PodRunning), "status", "phase") + Expect(err).NotTo(HaveOccurred()) + + Expect(cl.Status().Update(ctx, u)).To(Succeed()) + + actual := &corev1.Pod{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), actual)).To(Succeed()) + obj.ResourceVersion = actual.ResourceVersion + // only the status mutation should persist + obj.Status.Phase = corev1.PodRunning + Expect(cmp.Diff(obj, actual)).To(BeEmpty()) + }) + + It("should not change the status of unstructured objects that are configured to have a status subresource on patch", func(ctx SpecContext) { + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("foo/v1") + obj.SetKind("Foo") + obj.SetName("a-foo") + cl := NewClientBuilder().WithStatusSubresource(obj).Build() + + Expect(cl.Create(ctx, obj)).To(Succeed()) + original := obj.DeepCopy() + + err := unstructured.SetNestedField(obj.Object, map[string]interface{}{"count": int64(2)}, "status") + Expect(err).ToNot(HaveOccurred()) + Expect(cl.Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + + Expect(obj.Object["status"]).To(BeNil()) + + }) + + It("should not change non-status fields of unstructured objects that are configured to have a status subresource on status patch", func(ctx SpecContext) { obj := &unstructured.Unstructured{} obj.SetAPIVersion("foo/v1") obj.SetKind("Foo") @@ -1603,14 +2039,14 @@ var _ = Describe("Fake client", func() { err = unstructured.SetNestedField(obj.Object, map[string]any{"state": "new"}, "status") Expect(err).ToNot(HaveOccurred()) - Expect(cl.Status().Patch(context.Background(), obj, client.MergeFrom(original))).To(Succeed()) - Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(obj), obj)).To(Succeed()) + Expect(cl.Status().Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) Expect(obj.Object["status"]).To(BeEquivalentTo(map[string]any{"state": "new"})) Expect(obj.Object["spec"]).To(BeEquivalentTo("original")) }) - It("should return not found on status update of resources that don't have a status subresource", func() { + It("should return not found on status update of resources that don't have a status subresource", func(ctx SpecContext) { obj := &unstructured.Unstructured{} obj.SetAPIVersion("foo/v1") obj.SetKind("Foo") @@ -1618,7 +2054,7 @@ var _ = Describe("Fake client", func() { cl := NewClientBuilder().WithObjects(obj).Build() - err := cl.Status().Update(context.Background(), obj) + err := cl.Status().Update(ctx, obj) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) @@ -1627,74 +2063,1258 @@ var _ = Describe("Fake client", func() { &policyv1.Eviction{}, } for _, tp := range evictionTypes { - It("should delete a pod through the eviction subresource", func() { + It("should delete a pod through the eviction subresource", func(ctx SpecContext) { pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} cl := NewClientBuilder().WithObjects(pod).Build() - err := cl.SubResource("eviction").Create(context.Background(), pod, tp) + err := cl.SubResource("eviction").Create(ctx, pod, tp) Expect(err).NotTo(HaveOccurred()) - err = cl.Get(context.Background(), client.ObjectKeyFromObject(pod), pod) + err = cl.Get(ctx, client.ObjectKeyFromObject(pod), pod) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return not found when attempting to evict a pod that doesn't exist", func() { + It("should return not found when attempting to evict a pod that doesn't exist", func(ctx SpecContext) { cl := NewClientBuilder().Build() pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} - err := cl.SubResource("eviction").Create(context.Background(), pod, tp) + err := cl.SubResource("eviction").Create(ctx, pod, tp) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return not found when attempting to evict something other than a pod", func() { + It("should return not found when attempting to evict something other than a pod", func(ctx SpecContext) { ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} cl := NewClientBuilder().WithObjects(ns).Build() - err := cl.SubResource("eviction").Create(context.Background(), ns, tp) + err := cl.SubResource("eviction").Create(ctx, ns, tp) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) - It("should return an error when using the wrong subresource", func() { + It("should return an error when using the wrong subresource", func(ctx SpecContext) { cl := NewClientBuilder().Build() - err := cl.SubResource("eviction-subresource").Create(context.Background(), &corev1.Namespace{}, tp) + err := cl.SubResource("eviction-subresource").Create(ctx, &corev1.Namespace{}, tp) Expect(err).To(HaveOccurred()) }) } - It("should error when creating an eviction with the wrong type", func() { + It("should error when creating an eviction with the wrong type", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + err := cl.SubResource("eviction").Create(ctx, &corev1.Pod{}, &corev1.Namespace{}) + Expect(apierrors.IsBadRequest(err)).To(BeTrue()) + }) + + It("should create a ServiceAccount token through the token subresource", func(ctx SpecContext) { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + cl := NewClientBuilder().WithObjects(sa).Build() + + tokenRequest := &authenticationv1.TokenRequest{} + err := cl.SubResource("token").Create(ctx, sa, tokenRequest) + Expect(err).NotTo(HaveOccurred()) + + Expect(tokenRequest.Status.Token).NotTo(Equal("")) + Expect(tokenRequest.Status.ExpirationTimestamp).NotTo(Equal(metav1.Time{})) + }) + + It("should return not found when creating a token for a ServiceAccount that doesn't exist", func(ctx SpecContext) { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + cl := NewClientBuilder().Build() + + err := cl.SubResource("token").Create(ctx, sa, &authenticationv1.TokenRequest{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + It("should error when creating a token with the wrong subresource type", func(ctx SpecContext) { cl := NewClientBuilder().Build() - err := cl.SubResource("eviction").Create(context.Background(), &corev1.Pod{}, &corev1.Namespace{}) + err := cl.SubResource("token").Create(ctx, &corev1.ServiceAccount{}, &corev1.Namespace{}) + Expect(err).To(HaveOccurred()) Expect(apierrors.IsBadRequest(err)).To(BeTrue()) }) -}) -var _ = Describe("Fake client builder", func() { - It("panics when an index with the same name and GroupVersionKind is registered twice", func() { - // We need any realistic GroupVersionKind, the choice of apps/v1 Deployment is arbitrary. - cb := NewClientBuilder().WithIndex(&appsv1.Deployment{}, - "test-name", - func(client.Object) []string { return nil }) + It("should error when creating a token with the wrong type", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + err := cl.SubResource("token").Create(ctx, &corev1.Secret{}, &authenticationv1.TokenRequest{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) - Expect(func() { - cb.WithIndex(&appsv1.Deployment{}, - "test-name", - func(client.Object) []string { return []string{"foo"} }) - }).To(Panic()) + It("should leave typemeta empty on typed get", func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + }}).Build() + + var pod corev1.Pod + Expect(cl.Get(ctx, client.ObjectKey{Namespace: "default", Name: "foo"}, &pod)).NotTo(HaveOccurred()) + + Expect(pod.TypeMeta).To(Equal(metav1.TypeMeta{})) }) - It("should wrap the fake client with an interceptor when WithInterceptorFuncs is called", func() { - var called bool - cli := NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ - Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - called = true - return nil + It("should leave typemeta empty on typed list", func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "foo", + }}).Build() + + var podList corev1.PodList + Expect(cl.List(ctx, &podList)).NotTo(HaveOccurred()) + Expect(podList.ListMeta).To(Equal(metav1.ListMeta{})) + Expect(podList.Items[0].TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("should allow concurrent patches to a configMap", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + ResourceVersion: "0", }, - }).Build() - err := cli.Get(context.Background(), client.ObjectKey{}, &corev1.Pod{}) - Expect(err).NotTo(HaveOccurred()) - Expect(called).To(BeTrue()) + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries) + + for i := range tries { + go func() { + defer wg.Done() + defer GinkgoRecover() + + newObj := obj.DeepCopy() + newObj.Data = map[string]string{"foo": strconv.Itoa(i)} + Expect(cl.Patch(ctx, newObj, client.MergeFrom(obj))).To(Succeed()) + }() + } + wg.Wait() + + // While the order is not deterministic, there must be $tries distinct updates + // that each increment the resource version by one + Expect(cl.Get(ctx, client.ObjectKey{Name: "foo"}, obj)).To(Succeed()) + Expect(obj.ResourceVersion).To(Equal(strconv.Itoa(tries))) + }) + + It("should not allow concurrent patches to a configMap if the patch contains a ResourceVersion", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + ResourceVersion: "0", + }, + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + wg := sync.WaitGroup{} + wg.Add(5) + + for i := range 5 { + go func() { + defer wg.Done() + defer GinkgoRecover() + + newObj := obj.DeepCopy() + newObj.ResourceVersion = "1" // include an invalid RV to cause a conflict + newObj.Data = map[string]string{"foo": strconv.Itoa(i)} + Expect(apierrors.IsConflict(cl.Patch(ctx, newObj, client.MergeFrom(obj)))).To(BeTrue()) + }() + } + wg.Wait() + }) + + It("should allow concurrent updates to an object that allows unconditionalUpdate if the incoming request has no RV", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + ResourceVersion: "0", + }, + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries) + + for i := range tries { + go func() { + defer wg.Done() + defer GinkgoRecover() + + newObj := obj.DeepCopy() + newObj.Data = map[string]string{"foo": strconv.Itoa(i)} + newObj.ResourceVersion = "" + Expect(cl.Update(ctx, newObj)).To(Succeed()) + }() + } + wg.Wait() + + // While the order is not deterministic, there must be $tries distinct updates + // that each increment the resource version by one + Expect(cl.Get(ctx, client.ObjectKey{Name: "foo"}, obj)).To(Succeed()) + Expect(obj.ResourceVersion).To(Equal(strconv.Itoa(tries))) + }) + + It("If a create races with an update for an object that allows createOnUpdate, the update should always succeed", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries * 2) + + for i := range tries { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + go func() { + defer wg.Done() + defer GinkgoRecover() + + // this may or may not succeed depending on if we win the race. Either is acceptable, + // but if it fails, it must fail due to an AlreadyExists. + err := cl.Create(ctx, obj.DeepCopy()) + if err != nil { + Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) + } + }() + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This must always succeed, regardless of the outcome of the create. + Expect(cl.Update(ctx, obj.DeepCopy())).To(Succeed()) + }() + } + + wg.Wait() + }) + + It("If a delete races with an update for an object that allows createOnUpdate, the update should always succeed", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const tries = 50 + wg := sync.WaitGroup{} + wg.Add(tries * 2) + + for i := range tries { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + Expect(cl.Create(ctx, obj.DeepCopy())).To(Succeed()) + + go func() { + defer wg.Done() + defer GinkgoRecover() + + Expect(cl.Delete(ctx, obj.DeepCopy())).To(Succeed()) + }() + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This must always succeed, regardless of if the delete came before or + // after us. + Expect(cl.Update(ctx, obj.DeepCopy())).To(Succeed()) + }() + } + + wg.Wait() + }) + + It("If a DeleteAllOf races with a delete, the DeleteAllOf should always succeed", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const objects = 50 + wg := sync.WaitGroup{} + wg.Add(objects) + + for i := range objects { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + Expect(cl.Create(ctx, obj.DeepCopy())).To(Succeed()) + } + + for i := range objects { + obj := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This may or may not succeed depending on if the DeleteAllOf is faster, + // but if it fails, it should be a not found. + err := cl.Delete(ctx, obj) + if err != nil { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } + }() + } + Expect(cl.DeleteAllOf(ctx, &corev1.Service{})).To(Succeed()) + + wg.Wait() + }) + + It("If an update races with a scale update, only one of them succeeds", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(appsv1.AddToScheme(scheme)).To(Succeed()) + + cl := NewClientBuilder().WithScheme(scheme).Build() + + const tries = 5000 + for i := range tries { + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: strconv.Itoa(i), + }, + } + Expect(cl.Create(ctx, dep)).To(Succeed()) + + wg := sync.WaitGroup{} + wg.Add(2) + var updateSucceeded bool + var scaleSucceeded bool + + go func() { + defer wg.Done() + defer GinkgoRecover() + + dep := dep.DeepCopy() + dep.Annotations = map[string]string{"foo": "bar"} + + // This may or may not fail. If it does fail, it must be a conflict. + err := cl.Update(ctx, dep) + if err != nil { + Expect(apierrors.IsConflict(err)).To(BeTrue()) + } else { + updateSucceeded = true + } + }() + + go func() { + defer wg.Done() + defer GinkgoRecover() + + // This may or may not fail. If it does fail, it must be a conflict. + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 10}} + err := cl.SubResource("scale").Update(ctx, dep.DeepCopy(), client.WithSubResourceBody(scale)) + if err != nil { + Expect(apierrors.IsConflict(err)).To(BeTrue()) + } else { + scaleSucceeded = true + } + }() + + wg.Wait() + Expect(updateSucceeded).ToNot(Equal(scaleSucceeded)) + } + + }) + + It("disallows scale subresources on unsupported built-in types", func(ctx SpecContext) { + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + Expect(apiextensions.AddToScheme(scheme)).To(Succeed()) + + obj := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + cl := NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 2}} + expectedErr := "unimplemented scale subresource for resource *v1.Pod" + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scale).Error()).To(Equal(expectedErr)) + Expect(cl.SubResource(subResourceScale).Update(ctx, obj, client.WithSubResourceBody(scale)).Error()).To(Equal(expectedErr)) + }) + + It("disallows scale subresources on non-existing objects", func(ctx SpecContext) { + obj := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](2), + }, + } + cl := NewClientBuilder().Build() + + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 2}} + expectedErr := "deployments.apps \"foo\" not found" + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scale).Error()).To(Equal(expectedErr)) + Expect(cl.SubResource(subResourceScale).Update(ctx, obj, client.WithSubResourceBody(scale)).Error()).To(Equal(expectedErr)) + }) + + It("clears typemeta from structured objects on create", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().Build() + Expect(cl.Create(ctx, obj)).To(Succeed()) + Expect(obj.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on update", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + Expect(cl.Update(ctx, obj)).To(Succeed()) + Expect(obj.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on patch", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + original := obj.DeepCopy() + obj.TypeMeta = metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + } + Expect(cl.Patch(ctx, obj, client.MergeFrom(original))).To(Succeed()) + Expect(obj.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on get", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + target := &corev1.ConfigMap{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(obj), target)).To(Succeed()) + Expect(target.TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("clears typemeta from structured objects on list", func(ctx SpecContext) { + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + cl := NewClientBuilder().WithObjects(obj).Build() + target := &corev1.ConfigMapList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + } + Expect(cl.List(ctx, target)).To(Succeed()) + Expect(target.TypeMeta).To(Equal(metav1.TypeMeta{})) + Expect(target.Items[0].TypeMeta).To(Equal(metav1.TypeMeta{})) + }) + + It("is threadsafe", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + + u := func() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("custom/v1") + u.SetKind("Version") + u.SetName("foo") + return u + } + + uList := func() *unstructured.UnstructuredList { + u := &unstructured.UnstructuredList{} + u.SetAPIVersion("custom/v1") + u.SetKind("Version") + + return u + } + + meta := func() *metav1.PartialObjectMetadata { + return &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "custom/v1", + Kind: "Version", + }, + } + } + metaList := func() *metav1.PartialObjectMetadataList { + return &metav1.PartialObjectMetadataList{ + TypeMeta: metav1.TypeMeta{ + + APIVersion: "custom/v1", + Kind: "Version", + }, + } + } + + pod := func() *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }} + } + + ops := []func(){ + func() { _ = cl.Create(ctx, u()) }, + func() { _ = cl.Get(ctx, client.ObjectKeyFromObject(u()), u()) }, + func() { _ = cl.Update(ctx, u()) }, + func() { _ = cl.Patch(ctx, u(), client.RawPatch(types.StrategicMergePatchType, []byte("foo"))) }, + func() { _ = cl.Delete(ctx, u()) }, + func() { _ = cl.DeleteAllOf(ctx, u(), client.HasLabels{"foo"}) }, + func() { _ = cl.List(ctx, uList()) }, + + func() { _ = cl.Create(ctx, meta()) }, + func() { _ = cl.Get(ctx, client.ObjectKeyFromObject(meta()), meta()) }, + func() { _ = cl.Update(ctx, meta()) }, + func() { _ = cl.Patch(ctx, meta(), client.RawPatch(types.StrategicMergePatchType, []byte("foo"))) }, + func() { _ = cl.Delete(ctx, meta()) }, + func() { _ = cl.DeleteAllOf(ctx, meta(), client.HasLabels{"foo"}) }, + func() { _ = cl.List(ctx, metaList()) }, + + func() { _ = cl.Create(ctx, pod()) }, + func() { _ = cl.Get(ctx, client.ObjectKeyFromObject(pod()), pod()) }, + func() { _ = cl.Update(ctx, pod()) }, + func() { _ = cl.Patch(ctx, pod(), client.RawPatch(types.StrategicMergePatchType, []byte("foo"))) }, + func() { _ = cl.Delete(ctx, pod()) }, + func() { _ = cl.DeleteAllOf(ctx, pod(), client.HasLabels{"foo"}) }, + func() { _ = cl.List(ctx, &corev1.PodList{}) }, + } + + wg := sync.WaitGroup{} + wg.Add(len(ops)) + for _, op := range ops { + go func() { + defer wg.Done() + op() + }() + } + + wg.Wait() + }) + + DescribeTable("mutating operations return the updated object", + func(ctx SpecContext, mutate func(ctx SpecContext) (*corev1.ConfigMap, error)) { + mutated, err := mutate(ctx) + Expect(err).NotTo(HaveOccurred()) + + var retrieved corev1.ConfigMap + Expect(cl.Get(ctx, client.ObjectKeyFromObject(mutated), &retrieved)).To(Succeed()) + + Expect(&retrieved).To(BeComparableTo(mutated)) + }, + + Entry("create", func(ctx SpecContext) (*corev1.ConfigMap, error) { + cl = NewClientBuilder().Build() + cm.ResourceVersion = "" + return cm, cl.Create(ctx, cm) + }), + Entry("update", func(ctx SpecContext) (*corev1.ConfigMap, error) { + cl = NewClientBuilder().WithObjects(cm).Build() + cm.Labels = map[string]string{"updated-label": "update-test"} + cm.Data["new-key"] = "new-value" + return cm, cl.Update(ctx, cm) + }), + Entry("patch", func(ctx SpecContext) (*corev1.ConfigMap, error) { + cl = NewClientBuilder().WithObjects(cm).Build() + original := cm.DeepCopy() + + cm.Labels = map[string]string{"updated-label": "update-test"} + cm.Data["new-key"] = "new-value" + return cm, cl.Patch(ctx, cm, client.MergeFrom(original)) + }), + Entry("Create through Apply", func(ctx SpecContext) (*corev1.ConfigMap, error) { + ac := corev1applyconfigurations.ConfigMap(cm.Name, cm.Namespace).WithData(cm.Data) + + cl = NewClientBuilder().Build() + Expect(cl.Apply(ctx, ac, client.FieldOwner("foo"))).To(Succeed()) + + serialized, err := json.Marshal(ac) + Expect(err).NotTo(HaveOccurred()) + + var cm corev1.ConfigMap + Expect(json.Unmarshal(serialized, &cm)).To(Succeed()) + + // ApplyConfigurations always have TypeMeta set as they do not support using the scheme + // to retrieve gvk. + cm.TypeMeta = metav1.TypeMeta{} + return &cm, nil + }), + Entry("Update through Apply", func(ctx SpecContext) (*corev1.ConfigMap, error) { + ac := corev1applyconfigurations.ConfigMap(cm.Name, cm.Namespace). + WithLabels(map[string]string{"updated-label": "update-test"}). + WithData(map[string]string{"new-key": "new-value"}) + + cl = NewClientBuilder().WithObjects(cm).Build() + Expect(cl.Apply(ctx, ac, client.FieldOwner("foo"))).To(Succeed()) + + serialized, err := json.Marshal(ac) + Expect(err).NotTo(HaveOccurred()) + + var cm corev1.ConfigMap + Expect(json.Unmarshal(serialized, &cm)).To(Succeed()) + + // ApplyConfigurations always have TypeMeta set as they do not support using the scheme + // to retrieve gvk. + cm.TypeMeta = metav1.TypeMeta{} + return &cm, nil + }), + ) + + It("supports server-side apply of a client-go resource", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetName("foo") + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "data")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "data")).To(Succeed()) + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"other": "data"})) + }) + + It("supports server-side apply of a custom resource", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("custom/v1") + obj.SetKind("FakeResource") + obj.SetName("foo") + result := obj.DeepCopy() + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "spec")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "spec")).To(Succeed()) + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"other": "data"})) + }) + + It("errors out when doing SSA with managedFields set", func(ctx SpecContext) { + cl := NewClientBuilder().WithReturnManagedFields().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetName("foo") + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "data")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + err := cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo")) //nolint:staticcheck // will be removed once client.Apply is removed + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("metadata.managedFields must be nil")) + }) + + It("supports server-side apply using a custom type converter", func(ctx SpecContext) { + cl := NewClientBuilder(). + WithTypeConverters(clientgoapplyconfigurations.NewTypeConverter(clientgoscheme.Scheme)). + Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("v1") + obj.SetKind("ConfigMap") + obj.SetName("foo") + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "data")).To(Succeed()) + + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "data")).To(Succeed()) + Expect(cl.Patch(ctx, obj, client.Apply, client.FieldOwner("foo"))).To(Succeed()) //nolint:staticcheck // will be removed once client.Apply is removed + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(Equal(map[string]string{"other": "data"})) + }) + + It("returns managedFields if configured to do so", func(ctx SpecContext) { + cl := NewClientBuilder().WithReturnManagedFields().Build() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-cm", + Namespace: "default", + }, + Data: map[string]string{ + "initial": "data", + }, + } + Expect(cl.Create(ctx, cm)).NotTo(HaveOccurred()) + Expect(cm.ManagedFields).NotTo(BeNil()) + + retrieved := &corev1.ConfigMap{} + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), retrieved)).NotTo(HaveOccurred()) + Expect(retrieved.ManagedFields).NotTo(BeNil()) + + cm.Data["another"] = "value" + cm.SetManagedFields(nil) + Expect(cl.Update(ctx, cm)).NotTo(HaveOccurred()) + Expect(cm.ManagedFields).NotTo(BeNil()) + + cm.SetManagedFields(nil) + beforePatch := cm.DeepCopy() + cm.Data["a-third"] = "value" + Expect(cl.Patch(ctx, cm, client.MergeFrom(beforePatch))).NotTo(HaveOccurred()) + Expect(cm.ManagedFields).NotTo(BeNil()) + + u := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]any{ + "name": cm.Name, + "namespace": cm.Namespace, + }, + "data": map[string]any{ + "ssa": "value", + }, + }} + Expect(cl.Patch(ctx, u, client.Apply, client.FieldOwner("foo"))).NotTo(HaveOccurred()) //nolint:staticcheck // will be removed once client.Apply is removed + _, exists, err := unstructured.NestedFieldNoCopy(u.Object, "metadata", "managedFields") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + + var list corev1.ConfigMapList + Expect(cl.List(ctx, &list)).NotTo(HaveOccurred()) + for _, item := range list.Items { + Expect(item.ManagedFields).NotTo(BeNil()) + } + }) + + It("clears managedFields from objects in a list", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + Expect(cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + var list corev1.ConfigMapList + Expect(cl.List(ctx, &list)).NotTo(HaveOccurred()) + for _, item := range list.Items { + Expect(item.ManagedFields).To(BeNil()) + } + }) + + It("supports server-side apply of a client-go resource via Apply method", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + Expect(cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}} + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(BeComparableTo(map[string]string{"some": "data"})) + + obj.Data = map[string]string{"other": "data"} + Expect(cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(cm), cm)).To(Succeed()) + Expect(cm.Data).To(BeComparableTo(map[string]string{"other": "data"})) + }) + + It("returns a conflict when trying to Create an object with UID set through Apply", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithUID("123") + + err := cl.Apply(ctx, obj, &client.ApplyOptions{FieldManager: "test-manager"}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsConflict(err)).To(BeTrue()) + }) + + It("errors when trying to server-side apply an object without configuring a FieldManager", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + err := cl.Apply(ctx, obj) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "Expected error to be an invalid error") + }) + + It("errors when trying to server-side apply an object with an invalid FieldManager", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + err := cl.Apply(ctx, obj, client.FieldOwner("\x00")) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "Expected error to be an invalid error") + }) + + It("supports server-side apply of a custom resource via Apply method", func(ctx SpecContext) { + cl := NewClientBuilder().Build() + obj := &unstructured.Unstructured{} + obj.SetAPIVersion("custom/v1") + obj.SetKind("FakeResource") + obj.SetName("foo") + result := obj.DeepCopy() + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"some": "data"}, "spec")).To(Succeed()) + + applyConfig := client.ApplyConfigurationFromUnstructured(obj) + Expect(cl.Apply(ctx, applyConfig, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"some": "data"})) + + Expect(unstructured.SetNestedField(obj.Object, map[string]any{"other": "data"}, "spec")).To(Succeed()) + applyConfig2 := client.ApplyConfigurationFromUnstructured(obj) + Expect(cl.Apply(ctx, applyConfig2, &client.ApplyOptions{FieldManager: "test-manager"})).To(Succeed()) + + Expect(cl.Get(ctx, client.ObjectKeyFromObject(result), result)).To(Succeed()) + Expect(result.Object["spec"]).To(Equal(map[string]any{"other": "data"})) + }) + + It("sets the fieldManager in create, patch and update", func(ctx SpecContext) { + owner := "test-owner" + cl := client.WithFieldOwner( + NewClientBuilder().WithReturnManagedFields().Build(), + owner, + ) + + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "foo"}, + Data: map[string]string{"method": "create"}, + } + Expect(cl.Create(ctx, obj)).NotTo(HaveOccurred()) + + Expect(obj.ManagedFields).NotTo(BeEmpty()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + + originalObj := obj.DeepCopy() + obj.Data["method"] = "patch" + Expect(cl.Patch(ctx, obj, client.MergeFrom(originalObj))).NotTo(HaveOccurred()) + Expect(obj.ManagedFields).NotTo(BeEmpty()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + + obj.Data["method"] = "update" + Expect(cl.Update(ctx, obj)).NotTo(HaveOccurred()) + Expect(obj.ManagedFields).NotTo(BeEmpty()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + }) + + It("sets the fieldManager when creating through update", func(ctx SpecContext) { + owner := "test-owner" + cl := client.WithFieldOwner( + NewClientBuilder().WithReturnManagedFields().Build(), + owner, + ) + + obj := &corev1.Event{ObjectMeta: metav1.ObjectMeta{Name: "foo"}} + Expect(cl.Update(ctx, obj, client.FieldOwner(owner))).NotTo(HaveOccurred()) + for _, f := range obj.ManagedFields { + Expect(f.Manager).To(BeEquivalentTo(owner)) + } + }) + + // GH-3267 + It("Doesn't leave stale data when updating an object through SSA", func(ctx SpecContext) { + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + + obj.WithData(map[string]string{"bar": "baz"}) + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + var cms corev1.ConfigMapList + Expect(cl.List(ctx, &cms)).NotTo(HaveOccurred()) + Expect(len(cms.Items)).To(BeEquivalentTo(1)) + }) + + It("sets resourceVersion on SSA create", func(ctx SpecContext) { + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + // Ideally we should only test for it to not be empty, realistically we will + // break ppl if we ever start setting a different value. + Expect(obj.ResourceVersion).To(BeEquivalentTo(ptr.To("1"))) + }) + + It("ignores a passed resourceVersion on SSA create", func(ctx SpecContext) { + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithData(map[string]string{"some": "data"}). + WithResourceVersion("1234") + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + Expect(obj.ResourceVersion).To(BeEquivalentTo(ptr.To("1"))) + }) + + It("allows to set deletionTimestamp on an object during SSA create", func(ctx SpecContext) { + now := metav1.Time{Time: time.Now().Round(time.Second)} + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithDeletionTimestamp(now). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + + Expect(obj.DeletionTimestamp).To(BeEquivalentTo(&now)) + }) + + It("will silently ignore a deletionTimestamp update through SSA", func(ctx SpecContext) { + now := metav1.Time{Time: time.Now().Round(time.Second)} + obj := corev1applyconfigurations. + ConfigMap("foo", "default"). + WithDeletionTimestamp(now). + WithFinalizers("foo.bar"). + WithData(map[string]string{"some": "data"}) + + cl := NewClientBuilder().Build() + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + Expect(obj.DeletionTimestamp).To(BeEquivalentTo(&now)) + + later := metav1.Time{Time: now.Add(time.Second)} + obj.DeletionTimestamp = &later + Expect(cl.Apply(ctx, obj, client.FieldOwner("foo"))).NotTo(HaveOccurred()) + Expect(*obj.DeletionTimestamp).To(BeEquivalentTo(now)) + }) + + It("will error out if an object with invalid managedFields is added", func(ctx SpecContext) { + fieldV1Map := map[string]interface{}{ + "f:metadata": map[string]interface{}{ + "f:name": map[string]interface{}{}, + "f:labels": map[string]interface{}{}, + "f:annotations": map[string]interface{}{}, + "f:finalizers": map[string]interface{}{}, + }, + } + fieldV1, err := json.Marshal(fieldV1Map) + Expect(err).NotTo(HaveOccurred()) + + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", + Namespace: "default", + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "my-manager", + Operation: metav1.ManagedFieldsOperationUpdate, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{Raw: fieldV1}, + }}, + }} + + Expect(func() { + NewClientBuilder().WithObjects(obj).Build() + }).To(PanicWith(MatchError(ContainSubstring("invalid managedFields")))) + }) + + It("allows adding an object with managedFields", func(ctx SpecContext) { + fieldV1Map := map[string]interface{}{ + "f:metadata": map[string]interface{}{ + "f:name": map[string]interface{}{}, + "f:labels": map[string]interface{}{}, + "f:annotations": map[string]interface{}{}, + "f:finalizers": map[string]interface{}{}, + }, + } + fieldV1, err := json.Marshal(fieldV1Map) + Expect(err).NotTo(HaveOccurred()) + + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", + Namespace: "default", + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "my-manager", + Operation: metav1.ManagedFieldsOperationUpdate, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{Raw: fieldV1}, + APIVersion: "v1", + }}, + }} + + NewClientBuilder().WithObjects(obj).Build() + }) + + It("allows adding an object with invalid managedFields when not using the FieldManagedObjectTracker", func(ctx SpecContext) { + fieldV1Map := map[string]interface{}{ + "f:metadata": map[string]interface{}{ + "f:name": map[string]interface{}{}, + "f:labels": map[string]interface{}{}, + "f:annotations": map[string]interface{}{}, + "f:finalizers": map[string]interface{}{}, + }, + } + fieldV1, err := json.Marshal(fieldV1Map) + Expect(err).NotTo(HaveOccurred()) + + obj := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: "cm-1", + Namespace: "default", + ManagedFields: []metav1.ManagedFieldsEntry{{ + Manager: "my-manager", + Operation: metav1.ManagedFieldsOperationUpdate, + FieldsType: "FieldsV1", + FieldsV1: &metav1.FieldsV1{Raw: fieldV1}, + }}, + }} + + NewClientBuilder(). + WithObjectTracker(testing.NewObjectTracker( + clientgoscheme.Scheme, + serializer.NewCodecFactory(clientgoscheme.Scheme).UniversalDecoder(), + )). + WithObjects(obj). + Build() + }) + + scalableObjs := []client.Object{ + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: ptr.To[int32](2), + }, + }, + &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: ptr.To[int32](2), + }, + }, + &corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: corev1.ReplicationControllerSpec{ + Replicas: ptr.To[int32](2), + }, + }, + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To[int32](2), + }, + }, + } + for _, obj := range scalableObjs { + It(fmt.Sprintf("should be able to Get scale subresources for resource %T", obj), func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(obj).Build() + + scaleActual := &autoscalingv1.Scale{} + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scaleActual)).NotTo(HaveOccurred()) + + scaleExpected := &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Name: obj.GetName(), + UID: obj.GetUID(), + ResourceVersion: obj.GetResourceVersion(), + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: 2, + }, + } + Expect(cmp.Diff(scaleExpected, scaleActual)).To(BeEmpty()) + }) + + It(fmt.Sprintf("should be able to Update scale subresources for resource %T", obj), func(ctx SpecContext) { + cl := NewClientBuilder().WithObjects(obj).Build() + + scaleExpected := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 3}} + Expect(cl.SubResource(subResourceScale).Update(ctx, obj, client.WithSubResourceBody(scaleExpected))).NotTo(HaveOccurred()) + + objActual := obj.DeepCopyObject().(client.Object) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(objActual), objActual)).To(Succeed()) + + objExpected := obj.DeepCopyObject().(client.Object) + switch expected := objExpected.(type) { + case *appsv1.Deployment: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + case *appsv1.ReplicaSet: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + case *corev1.ReplicationController: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + case *appsv1.StatefulSet: + expected.ResourceVersion = objActual.GetResourceVersion() + expected.Spec.Replicas = ptr.To(int32(3)) + } + Expect(cmp.Diff(objExpected, objActual)).To(BeEmpty()) + + scaleActual := &autoscalingv1.Scale{} + Expect(cl.SubResource(subResourceScale).Get(ctx, obj, scaleActual)).NotTo(HaveOccurred()) + + // When we called Update, these were derived but we need them now to compare. + scaleExpected.Name = scaleActual.Name + scaleExpected.ResourceVersion = scaleActual.ResourceVersion + Expect(cmp.Diff(scaleExpected, scaleActual)).To(BeEmpty()) + }) + + } +}) + +type Schemaless map[string]interface{} + +type WithSchemalessSpec struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec Schemaless `json:"spec,omitempty"` +} + +func (t *WithSchemalessSpec) DeepCopy() *WithSchemalessSpec { + w := &WithSchemalessSpec{ + ObjectMeta: *t.ObjectMeta.DeepCopy(), + } + w.TypeMeta = metav1.TypeMeta{ + APIVersion: t.APIVersion, + Kind: t.Kind, + } + t.Spec.DeepCopyInto(&w.Spec) + + return w +} + +func (t *WithSchemalessSpec) DeepCopyObject() runtime.Object { + return t.DeepCopy() +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Schemaless) DeepCopyInto(out *Schemaless) { + if *in != nil { + *out = make(Schemaless, len(*in)) + for key := range *in { + (*out)[key] = (*in)[key] + } + } +} + +// DeepCopy copies the receiver, creating a new Schemaless. +func (in *Schemaless) DeepCopy() *Schemaless { + if in == nil { + return nil + } + out := new(Schemaless) + in.DeepCopyInto(out) + return out +} + +var _ = Describe("Fake client builder", func() { + It("panics when an index with the same name and GroupVersionKind is registered twice", func(ctx SpecContext) { + // We need any realistic GroupVersionKind, the choice of apps/v1 Deployment is arbitrary. + cb := NewClientBuilder().WithIndex(&appsv1.Deployment{}, + "test-name", + func(client.Object) []string { return nil }) + + Expect(func() { + cb.WithIndex(&appsv1.Deployment{}, + "test-name", + func(client.Object) []string { return []string{"foo"} }) + }).To(Panic()) + }) + + It("should wrap the fake client with an interceptor when WithInterceptorFuncs is called", func(ctx SpecContext) { + var called bool + cli := NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + called = true + return nil + }, + }).Build() + err := cli.Get(ctx, client.ObjectKey{}, &corev1.Pod{}) + Expect(err).NotTo(HaveOccurred()) + Expect(called).To(BeTrue()) + }) + + It("should panic when calling build more than once", func() { + cb := NewClientBuilder() + anotherCb := cb + cb.Build() + Expect(func() { + anotherCb.Build() + }).To(Panic()) }) }) diff --git a/pkg/client/fake/doc.go b/pkg/client/fake/doc.go index d42347a2e2..47cad3980d 100644 --- a/pkg/client/fake/doc.go +++ b/pkg/client/fake/doc.go @@ -20,7 +20,7 @@ Package fake provides a fake client for testing. A fake client is backed by its simple object store indexed by GroupVersionResource. You can create a fake client with optional objects. - client := NewClientBuilder().WithScheme(scheme).WithObj(initObjs...).Build() + client := NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() You can invoke the methods defined in the Client interface. diff --git a/pkg/client/fake/typeconverter.go b/pkg/client/fake/typeconverter.go new file mode 100644 index 0000000000..3cb3a0dc77 --- /dev/null +++ b/pkg/client/fake/typeconverter.go @@ -0,0 +1,60 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/managedfields" + "sigs.k8s.io/structured-merge-diff/v6/typed" +) + +type multiTypeConverter struct { + upstream []managedfields.TypeConverter +} + +func (m multiTypeConverter) ObjectToTyped(r runtime.Object, o ...typed.ValidationOptions) (*typed.TypedValue, error) { + var errs []error + for _, u := range m.upstream { + res, err := u.ObjectToTyped(r, o...) + if err != nil { + errs = append(errs, err) + continue + } + + return res, nil + } + + return nil, fmt.Errorf("failed to convert Object to TypedValue: %w", kerrors.NewAggregate(errs)) +} + +func (m multiTypeConverter) TypedToObject(v *typed.TypedValue) (runtime.Object, error) { + var errs []error + for _, u := range m.upstream { + res, err := u.TypedToObject(v) + if err != nil { + errs = append(errs, err) + continue + } + + return res, nil + } + + return nil, fmt.Errorf("failed to convert TypedValue to Object: %w", kerrors.NewAggregate(errs)) +} diff --git a/pkg/client/fake/typeconverter_test.go b/pkg/client/fake/typeconverter_test.go new file mode 100644 index 0000000000..8acba79f88 --- /dev/null +++ b/pkg/client/fake/typeconverter_test.go @@ -0,0 +1,200 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/managedfields" + "sigs.k8s.io/structured-merge-diff/v6/typed" +) + +var _ = Describe("multiTypeConverter", func() { + Describe("ObjectToTyped", func() { + It("should use first converter when it succeeds", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + testTyped := &typed.TypedValue{} + + firstConverter := &mockTypeConverter{ + objectToTypedResult: testTyped, + } + secondConverter := &mockTypeConverter{ + objectToTypedError: errors.New("second converter should not be called"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testTyped)) + }) + + It("should use second converter when first fails", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + testTyped := &typed.TypedValue{} + + firstConverter := &mockTypeConverter{ + objectToTypedError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + objectToTypedResult: testTyped, + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testTyped)) + }) + + It("should return aggregate error when all converters fail", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + firstConverter := &mockTypeConverter{ + objectToTypedError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + objectToTypedError: errors.New("second converter error"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert Object to Typed")) + Expect(err.Error()).To(ContainSubstring("first converter error")) + Expect(err.Error()).To(ContainSubstring("second converter error")) + }) + + It("should return error when no converters provided", func() { + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{}, + } + + result, err := converter.ObjectToTyped(testObj) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert Object to Typed")) + }) + }) + + Describe("TypedToObject", func() { + It("should use first converter when it succeeds", func() { + testTyped := &typed.TypedValue{} + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + firstConverter := &mockTypeConverter{ + typedToObjectResult: testObj, + } + secondConverter := &mockTypeConverter{ + typedToObjectError: errors.New("second converter should not be called"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testObj)) + }) + + It("should use second converter when first fails", func() { + testTyped := &typed.TypedValue{} + testObj := &corev1.ConfigMap{Data: map[string]string{"key": "value"}} + + firstConverter := &mockTypeConverter{ + typedToObjectError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + typedToObjectResult: testObj, + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(testObj)) + }) + + It("should return aggregate error when all converters fail", func() { + testTyped := &typed.TypedValue{} + + firstConverter := &mockTypeConverter{ + typedToObjectError: errors.New("first converter error"), + } + secondConverter := &mockTypeConverter{ + typedToObjectError: errors.New("second converter error"), + } + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{firstConverter, secondConverter}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert TypedValue to Object")) + Expect(err.Error()).To(ContainSubstring("first converter error")) + Expect(err.Error()).To(ContainSubstring("second converter error")) + }) + + It("should return error when no converters provided", func() { + testTyped := &typed.TypedValue{} + + converter := multiTypeConverter{ + upstream: []managedfields.TypeConverter{}, + } + + result, err := converter.TypedToObject(testTyped) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + Expect(err.Error()).To(ContainSubstring("failed to convert TypedValue to Object")) + }) + }) +}) + +type mockTypeConverter struct { + objectToTypedResult *typed.TypedValue + objectToTypedError error + + typedToObjectResult runtime.Object + typedToObjectError error +} + +func (m *mockTypeConverter) ObjectToTyped(r runtime.Object, o ...typed.ValidationOptions) (*typed.TypedValue, error) { + return m.objectToTypedResult, m.objectToTypedError +} + +func (m *mockTypeConverter) TypedToObject(v *typed.TypedValue) (runtime.Object, error) { + return m.typedToObjectResult, m.typedToObjectError +} diff --git a/pkg/client/fake/versioned_tracker.go b/pkg/client/fake/versioned_tracker.go new file mode 100644 index 0000000000..bc1eaeb951 --- /dev/null +++ b/pkg/client/fake/versioned_tracker.go @@ -0,0 +1,361 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "bytes" + "errors" + "fmt" + "runtime/debug" + "strconv" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/managedfields" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +var _ testing.ObjectTracker = (*versionedTracker)(nil) + +type versionedTracker struct { + upstream testing.ObjectTracker + scheme *runtime.Scheme + withStatusSubresource sets.Set[schema.GroupVersionKind] + usesFieldManagedObjectTracker bool +} + +func (t versionedTracker) Add(obj runtime.Object) error { + var objects []runtime.Object + if meta.IsListType(obj) { + var err error + objects, err = meta.ExtractList(obj) + if err != nil { + return err + } + } else { + objects = []runtime.Object{obj} + } + for _, obj := range objects { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetDeletionTimestamp() != nil && len(accessor.GetFinalizers()) == 0 { + return fmt.Errorf("refusing to create obj %s with metadata.deletionTimestamp but no finalizers", accessor.GetName()) + } + if accessor.GetResourceVersion() == "" { + // We use a "magic" value of 999 here because this field + // is parsed as uint and and 0 is already used in Update. + // As we can't go lower, go very high instead so this can + // be recognized + accessor.SetResourceVersion(trackerAddResourceVersion) + } + + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + + // If the fieldManager can not decode fields, it will just silently clear them. This is pretty + // much guaranteed not to be what someone that initializes a fake client with objects that + // have them set wants, so validate them here. + // Ref https://github.com/kubernetes/kubernetes/blob/a956ef4862993b825bcd524a19260192ff1da72d/staging/src/k8s.io/apimachinery/pkg/util/managedfields/internal/fieldmanager.go#L105 + if t.usesFieldManagedObjectTracker { + if err := managedfields.ValidateManagedFields(accessor.GetManagedFields()); err != nil { + return fmt.Errorf("invalid managedFields on %T: %w", obj, err) + } + } + if err := t.upstream.Add(obj); err != nil { + return err + } + } + + return nil +} + +func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.CreateOptions) error { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetName() == "" { + gvk, _ := apiutil.GVKForObject(obj, t.scheme) + return apierrors.NewInvalid( + gvk.GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + if accessor.GetResourceVersion() != "" { + return apierrors.NewBadRequest("resourceVersion can not be set for Create requests") + } + accessor.SetResourceVersion("1") + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + if err := t.upstream.Create(gvr, obj, ns, opts...); err != nil { + accessor.SetResourceVersion("") + return err + } + + return nil +} + +func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.UpdateOptions) error { + updateOpts, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + + return t.update(gvr, obj, ns, false, false, updateOpts) +} + +func (t versionedTracker) update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, isStatus, deleting bool, opts metav1.UpdateOptions) error { + gvk, err := apiutil.GVKForObject(obj, t.scheme) + if err != nil { + return err + } + obj, needsCreate, err := t.updateObject(gvr, gvk, obj, ns, isStatus, deleting, allowsCreateOnUpdate(gvk), opts.DryRun) + if err != nil { + return err + } + + if needsCreate { + opts := metav1.CreateOptions{DryRun: opts.DryRun, FieldManager: opts.FieldManager} + return t.Create(gvr, obj, ns, opts) + } + + if obj == nil { // Object was deleted in updateObject + return nil + } + + if u, unstructured := obj.(*unstructured.Unstructured); unstructured { + u.SetGroupVersionKind(gvk) + } + + return t.upstream.Update(gvr, obj, ns, opts) +} + +func (t versionedTracker) Patch(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.PatchOptions) error { + patchOptions, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + + gvk, err := apiutil.GVKForObject(obj, t.scheme) + if err != nil { + return err + } + + // We apply patches using a client-go reaction that ends up calling the trackers Patch. As we can't change + // that reaction, we use the callstack to figure out if this originated from the status client. + isStatus := bytes.Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeSubResourceClient).statusPatch")) + + obj, needsCreate, err := t.updateObject(gvr, gvk, obj, ns, isStatus, false, allowsCreateOnUpdate(gvk), patchOptions.DryRun) + if err != nil { + return err + } + if needsCreate { + opts := metav1.CreateOptions{DryRun: patchOptions.DryRun, FieldManager: patchOptions.FieldManager} + return t.Create(gvr, obj, ns, opts) + } + + if obj == nil { // Object was deleted in updateObject + return nil + } + + return t.upstream.Patch(gvr, obj, ns, patchOptions) +} + +// updateObject performs a number of validations and changes related to +// object updates, such as checking and updating the resourceVersion. +func (t versionedTracker) updateObject( + gvr schema.GroupVersionResource, + gvk schema.GroupVersionKind, + obj runtime.Object, + ns string, + isStatus bool, + deleting bool, + allowCreateOnUpdate bool, + dryRun []string, +) (result runtime.Object, needsCreate bool, _ error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, false, fmt.Errorf("failed to get accessor for object: %w", err) + } + + if accessor.GetName() == "" { + return nil, false, apierrors.NewInvalid( + gvk.GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + + oldObject, err := t.Get(gvr, ns, accessor.GetName()) + if err != nil { + // If the resource is not found and the resource allows create on update, issue a + // create instead. + if apierrors.IsNotFound(err) && allowCreateOnUpdate { + // Pass this info to the caller rather than create, because in the SSA case it + // must be created by calling Apply in the upstream tracker, not Create. + // This is because SSA considers Apply and Non-Apply operations to be different + // even when they use the same fieldManager. This behavior is also observable + // with a real Kubernetes apiserver. + // + // Ref https://kubernetes.slack.com/archives/C0EG7JC6T/p1757868204458989?thread_ts=1757808656.002569&cid=C0EG7JC6T + return obj, true, nil + } + return obj, false, err + } + + if t.withStatusSubresource.Has(gvk) { + if isStatus { // copy everything but status and metadata.ResourceVersion from original object + if err := copyStatusFrom(obj, oldObject); err != nil { + return nil, false, fmt.Errorf("failed to copy non-status field for object with status subresouce: %w", err) + } + passedRV := accessor.GetResourceVersion() + if err := copyFrom(oldObject, obj); err != nil { + return nil, false, fmt.Errorf("failed to restore non-status fields: %w", err) + } + accessor.SetResourceVersion(passedRV) + } else { // copy status from original object + if err := copyStatusFrom(oldObject, obj); err != nil { + return nil, false, fmt.Errorf("failed to copy the status for object with status subresource: %w", err) + } + } + } else if isStatus { + return nil, false, apierrors.NewNotFound(gvr.GroupResource(), accessor.GetName()) + } + + oldAccessor, err := meta.Accessor(oldObject) + if err != nil { + return nil, false, err + } + + // If the new object does not have the resource version set and it allows unconditional update, + // default it to the resource version of the existing resource + if accessor.GetResourceVersion() == "" { + switch { + case allowsUnconditionalUpdate(gvk): + accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) + // This is needed because if the patch explicitly sets the RV to null, the client-go reaction we use + // to apply it and whose output we process here will have it unset. It is not clear why the Kubernetes + // apiserver accepts such a patch, but it does so we just copy that behavior. + // Kubernetes apiserver behavior can be checked like this: + // `kubectl patch configmap foo --patch '{"metadata":{"annotations":{"foo":"bar"},"resourceVersion":null}}' -v=9` + case bytes. + Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeClient).Patch")): + // We apply patches using a client-go reaction that ends up calling the trackers Update. As we can't change + // that reaction, we use the callstack to figure out if this originated from the "fakeClient.Patch" func. + accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) + } + } + + if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() { + return nil, false, apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified")) + } + if oldAccessor.GetResourceVersion() == "" { + oldAccessor.SetResourceVersion("0") + } + intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64) + if err != nil { + return nil, false, fmt.Errorf("can not convert resourceVersion %q to int: %w", oldAccessor.GetResourceVersion(), err) + } + intResourceVersion++ + accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10)) + + if !deleting && !deletionTimestampEqual(accessor, oldAccessor) { + return nil, false, fmt.Errorf("error: Unable to edit %s: metadata.deletionTimestamp field is immutable", accessor.GetName()) + } + + if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 { + return nil, false, t.Delete(gvr, accessor.GetNamespace(), accessor.GetName(), metav1.DeleteOptions{DryRun: dryRun}) + } + + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + return obj, false, err +} + +func (t versionedTracker) Apply(gvr schema.GroupVersionResource, applyConfiguration runtime.Object, ns string, opts ...metav1.PatchOptions) error { + patchOptions, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + gvk, err := apiutil.GVKForObject(applyConfiguration, t.scheme) + if err != nil { + return err + } + isStatus := bytes.Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeSubResourceClient).statusPatch")) + + applyConfiguration, needsCreate, err := t.updateObject(gvr, gvk, applyConfiguration, ns, isStatus, false, true, patchOptions.DryRun) + if err != nil { + return err + } + + if needsCreate { + // https://github.com/kubernetes/kubernetes/blob/81affffa1b8d8079836f4cac713ea8d1b2bbf10f/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/patch.go#L606 + accessor, err := meta.Accessor(applyConfiguration) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetUID() != "" { + return apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), fmt.Errorf("uid mismatch: the provided object specified uid %s, and no existing object was found", accessor.GetUID())) + } + + if t.withStatusSubresource.Has(gvk) { + // Clear out status for create, for update this is handled in updateObject + if err := copyStatusFrom(&unstructured.Unstructured{}, applyConfiguration); err != nil { + return err + } + } + } + + if applyConfiguration == nil { // Object was deleted in updateObject + return nil + } + + if isStatus { + // We restore everything but status from the tracker where we don't put GVK + // into the object but it must be set for the ManagedFieldsObjectTracker + applyConfiguration.GetObjectKind().SetGroupVersionKind(gvk) + } + return t.upstream.Apply(gvr, applyConfiguration, ns, opts...) +} + +func (t versionedTracker) Delete(gvr schema.GroupVersionResource, ns, name string, opts ...metav1.DeleteOptions) error { + return t.upstream.Delete(gvr, ns, name, opts...) +} + +func (t versionedTracker) Get(gvr schema.GroupVersionResource, ns, name string, opts ...metav1.GetOptions) (runtime.Object, error) { + return t.upstream.Get(gvr, ns, name, opts...) +} + +func (t versionedTracker) List(gvr schema.GroupVersionResource, gvk schema.GroupVersionKind, ns string, opts ...metav1.ListOptions) (runtime.Object, error) { + return t.upstream.List(gvr, gvk, ns, opts...) +} + +func (t versionedTracker) Watch(gvr schema.GroupVersionResource, ns string, opts ...metav1.ListOptions) (watch.Interface, error) { + return t.upstream.Watch(gvr, ns, opts...) +} diff --git a/pkg/client/fieldowner.go b/pkg/client/fieldowner.go new file mode 100644 index 0000000000..5d9437ba91 --- /dev/null +++ b/pkg/client/fieldowner.go @@ -0,0 +1,114 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// WithFieldOwner wraps a Client and adds the fieldOwner as the field +// manager to all write requests from this client. If additional [FieldOwner] +// options are specified on methods of this client, the value specified here +// will be overridden. +func WithFieldOwner(c Client, fieldOwner string) Client { + return &clientWithFieldManager{ + owner: fieldOwner, + c: c, + Reader: c, + } +} + +type clientWithFieldManager struct { + owner string + c Client + Reader +} + +func (f *clientWithFieldManager) Create(ctx context.Context, obj Object, opts ...CreateOption) error { + return f.c.Create(ctx, obj, append([]CreateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { + return f.c.Update(ctx, obj, append([]UpdateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { + return f.c.Patch(ctx, obj, patch, append([]PatchOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + return f.c.Apply(ctx, obj, append([]ApplyOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *clientWithFieldManager) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { + return f.c.Delete(ctx, obj, opts...) +} + +func (f *clientWithFieldManager) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { + return f.c.DeleteAllOf(ctx, obj, opts...) +} + +func (f *clientWithFieldManager) Scheme() *runtime.Scheme { return f.c.Scheme() } +func (f *clientWithFieldManager) RESTMapper() meta.RESTMapper { return f.c.RESTMapper() } +func (f *clientWithFieldManager) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return f.c.GroupVersionKindFor(obj) +} +func (f *clientWithFieldManager) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return f.c.IsObjectNamespaced(obj) +} + +func (f *clientWithFieldManager) Status() StatusWriter { + return &subresourceClientWithFieldOwner{ + owner: f.owner, + subresourceWriter: f.c.Status(), + } +} + +func (f *clientWithFieldManager) SubResource(subresource string) SubResourceClient { + c := f.c.SubResource(subresource) + return &subresourceClientWithFieldOwner{ + owner: f.owner, + subresourceWriter: c, + SubResourceReader: c, + } +} + +type subresourceClientWithFieldOwner struct { + owner string + subresourceWriter SubResourceWriter + SubResourceReader +} + +func (f *subresourceClientWithFieldOwner) Create(ctx context.Context, obj Object, subresource Object, opts ...SubResourceCreateOption) error { + return f.subresourceWriter.Create(ctx, obj, subresource, append([]SubResourceCreateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *subresourceClientWithFieldOwner) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + return f.subresourceWriter.Update(ctx, obj, append([]SubResourceUpdateOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *subresourceClientWithFieldOwner) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + return f.subresourceWriter.Patch(ctx, obj, patch, append([]SubResourcePatchOption{FieldOwner(f.owner)}, opts...)...) +} + +func (f *subresourceClientWithFieldOwner) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + return f.subresourceWriter.Apply(ctx, obj, append([]SubResourceApplyOption{FieldOwner(f.owner)}, opts...)...) +} diff --git a/pkg/client/fieldowner_test.go b/pkg/client/fieldowner_test.go new file mode 100644 index 0000000000..069abbc115 --- /dev/null +++ b/pkg/client/fieldowner_test.go @@ -0,0 +1,180 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client_test + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestWithFieldOwner(t *testing.T) { + calls := 0 + fakeClient := testClient(t, "custom-field-mgr", func() { calls++ }) + wrappedClient := client.WithFieldOwner(fakeClient, "custom-field-mgr") + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + dummyObjectAC := corev1applyconfigurations.Namespace(dummyObj.Name) + + _ = wrappedClient.Create(ctx, dummyObj) + _ = wrappedClient.Update(ctx, dummyObj) + _ = wrappedClient.Patch(ctx, dummyObj, nil) + _ = wrappedClient.Apply(ctx, dummyObjectAC) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.Status().Update(ctx, dummyObj) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil) + _ = wrappedClient.Status().Apply(ctx, dummyObjectAC) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil) + _ = wrappedClient.SubResource("some-subresource").Apply(ctx, dummyObjectAC) + + if expectedCalls := 12; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +func TestWithFieldOwnerOverridden(t *testing.T) { + calls := 0 + + fakeClient := testClient(t, "new-field-manager", func() { calls++ }) + wrappedClient := client.WithFieldOwner(fakeClient, "old-field-manager") + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + dummyObjectAC := corev1applyconfigurations.Namespace(dummyObj.Name) + + _ = wrappedClient.Create(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Update(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Patch(ctx, dummyObj, nil, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Apply(ctx, dummyObjectAC, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Update(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil, client.FieldOwner("new-field-manager")) + _ = wrappedClient.Status().Apply(ctx, dummyObjectAC, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil, client.FieldOwner("new-field-manager")) + _ = wrappedClient.SubResource("some-subresource").Apply(ctx, dummyObjectAC, client.FieldOwner("new-field-manager")) + + if expectedCalls := 12; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +// testClient is a helper function that checks if calls have the expected field manager, +// and calls the callback function on each intercepted call. +func testClient(t *testing.T, expectedFieldManager string, callback func()) client.Client { + // TODO: we could use the dummyClient in interceptor pkg if we move it to an internal pkg + return fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + callback() + out := &client.CreateOptions{} + for _, f := range opts { + f.ApplyToCreate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + callback() + out := &client.UpdateOptions{} + for _, f := range opts { + f.ApplyToUpdate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + callback() + out := &client.PatchOptions{} + for _, f := range opts { + f.ApplyToPatch(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + callback() + out := &client.SubResourceCreateOptions{} + for _, f := range opts { + f.ApplyToSubResourceCreate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + callback() + out := &client.SubResourceUpdateOptions{} + for _, f := range opts { + f.ApplyToSubResourceUpdate(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourcePatch: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + callback() + out := &client.SubResourcePatchOptions{} + for _, f := range opts { + f.ApplyToSubResourcePatch(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + Apply: func(ctx context.Context, c client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + callback() + out := &client.ApplyOptions{} + for _, f := range opts { + f.ApplyToApply(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + SubResourceApply: func(ctx context.Context, c client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + callback() + out := &client.SubResourceApplyOptions{} + for _, f := range opts { + f.ApplyToSubResourceApply(out) + } + if got := out.FieldManager; expectedFieldManager != got { + t.Fatalf("wrong field manager: expected=%q; got=%q", expectedFieldManager, got) + } + return nil + }, + }).Build() +} diff --git a/pkg/client/fieldvalidation.go b/pkg/client/fieldvalidation.go new file mode 100644 index 0000000000..b0f660854e --- /dev/null +++ b/pkg/client/fieldvalidation.go @@ -0,0 +1,117 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// WithFieldValidation wraps a Client and configures field validation, by +// default, for all write requests from this client. Users can override field +// validation for individual write requests. +// +// This wrapper has no effect on apply requests, as they do not support a +// custom fieldValidation setting, it is always strict. +func WithFieldValidation(c Client, validation FieldValidation) Client { + return &clientWithFieldValidation{ + validation: validation, + client: c, + Reader: c, + } +} + +type clientWithFieldValidation struct { + validation FieldValidation + client Client + Reader +} + +func (c *clientWithFieldValidation) Create(ctx context.Context, obj Object, opts ...CreateOption) error { + return c.client.Create(ctx, obj, append([]CreateOption{c.validation}, opts...)...) +} + +func (c *clientWithFieldValidation) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { + return c.client.Update(ctx, obj, append([]UpdateOption{c.validation}, opts...)...) +} + +func (c *clientWithFieldValidation) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { + return c.client.Patch(ctx, obj, patch, append([]PatchOption{c.validation}, opts...)...) +} + +func (c *clientWithFieldValidation) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + return c.client.Apply(ctx, obj, opts...) +} + +func (c *clientWithFieldValidation) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { + return c.client.Delete(ctx, obj, opts...) +} + +func (c *clientWithFieldValidation) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { + return c.client.DeleteAllOf(ctx, obj, opts...) +} + +func (c *clientWithFieldValidation) Scheme() *runtime.Scheme { return c.client.Scheme() } +func (c *clientWithFieldValidation) RESTMapper() meta.RESTMapper { return c.client.RESTMapper() } +func (c *clientWithFieldValidation) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.client.GroupVersionKindFor(obj) +} + +func (c *clientWithFieldValidation) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.client.IsObjectNamespaced(obj) +} + +func (c *clientWithFieldValidation) Status() StatusWriter { + return &subresourceClientWithFieldValidation{ + validation: c.validation, + subresourceWriter: c.client.Status(), + } +} + +func (c *clientWithFieldValidation) SubResource(subresource string) SubResourceClient { + srClient := c.client.SubResource(subresource) + return &subresourceClientWithFieldValidation{ + validation: c.validation, + subresourceWriter: srClient, + SubResourceReader: srClient, + } +} + +type subresourceClientWithFieldValidation struct { + validation FieldValidation + subresourceWriter SubResourceWriter + SubResourceReader +} + +func (c *subresourceClientWithFieldValidation) Create(ctx context.Context, obj Object, subresource Object, opts ...SubResourceCreateOption) error { + return c.subresourceWriter.Create(ctx, obj, subresource, append([]SubResourceCreateOption{c.validation}, opts...)...) +} + +func (c *subresourceClientWithFieldValidation) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + return c.subresourceWriter.Update(ctx, obj, append([]SubResourceUpdateOption{c.validation}, opts...)...) +} + +func (c *subresourceClientWithFieldValidation) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + return c.subresourceWriter.Patch(ctx, obj, patch, append([]SubResourcePatchOption{c.validation}, opts...)...) +} + +func (c *subresourceClientWithFieldValidation) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + return c.subresourceWriter.Apply(ctx, obj, opts...) +} diff --git a/pkg/client/fieldvalidation_test.go b/pkg/client/fieldvalidation_test.go new file mode 100644 index 0000000000..6e6e9e5d17 --- /dev/null +++ b/pkg/client/fieldvalidation_test.go @@ -0,0 +1,297 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +var _ = Describe("ClientWithFieldValidation", func() { + It("should return errors for invalid fields when using strict validation", func(ctx SpecContext) { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + wrappedClient := client.WithFieldValidation(cl, metav1.FieldValidationStrict) + + baseNode := &unstructured.Unstructured{} + baseNode.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Kind: "Node", + Version: "v1", + }) + baseNode.SetName("client-with-field-validation-test-node") + + validNode := baseNode.DeepCopy() + patch := client.MergeFrom(validNode.DeepCopy()) + + invalidNode := baseNode.DeepCopy() + Expect(unstructured.SetNestedField(invalidNode.Object, "value", "spec", "invalidField")).To(Succeed()) + + invalidStatusNode := baseNode.DeepCopy() + Expect(unstructured.SetNestedField(invalidStatusNode.Object, "value", "status", "invalidStatusField")).To(Succeed()) + + err = wrappedClient.Create(ctx, invalidNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"spec.invalidField\"")) + + err = wrappedClient.Create(ctx, validNode) + Expect(err).ToNot(HaveOccurred()) + + err = wrappedClient.Update(ctx, invalidNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"spec.invalidField\"")) + + err = wrappedClient.Patch(ctx, invalidNode, patch) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"spec.invalidField\"")) + + // Status.Create is not supported on Nodes + + err = wrappedClient.Status().Update(ctx, invalidStatusNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + err = wrappedClient.Status().Patch(ctx, invalidStatusNode, patch) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + // Status.Create is not supported on Nodes + + err = wrappedClient.SubResource("status").Update(ctx, invalidStatusNode) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + err = wrappedClient.SubResource("status").Patch(ctx, invalidStatusNode, patch) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("strict decoding error: unknown field \"status.invalidStatusField\"")) + + invalidApplyConfig := client.ApplyConfigurationFromUnstructured(invalidStatusNode) + err = wrappedClient.Status().Apply(ctx, invalidApplyConfig, client.FieldOwner("test-owner")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("field not declared in schema")) + + err = wrappedClient.SubResource("status").Apply(ctx, invalidApplyConfig, client.FieldOwner("test-owner")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("field not declared in schema")) + }) +}) + +func TestWithStrictFieldValidation(t *testing.T) { + calls := 0 + fakeClient := testFieldValidationClient(t, metav1.FieldValidationStrict, func() { calls++ }) + wrappedClient := client.WithFieldValidation(fakeClient, metav1.FieldValidationStrict) + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + + _ = wrappedClient.Create(ctx, dummyObj) + _ = wrappedClient.Update(ctx, dummyObj) + _ = wrappedClient.Apply(ctx, corev1applyconfigurations.ConfigMap("foo", "bar")) + _ = wrappedClient.Patch(ctx, dummyObj, nil) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.Status().Update(ctx, dummyObj) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil) + _ = wrappedClient.Status().Apply(ctx, corev1applyconfigurations.Namespace(""), nil) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil) + _ = wrappedClient.SubResource("some-subresource").Apply(ctx, corev1applyconfigurations.Namespace(""), nil) + + if expectedCalls := 12; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +func TestWithStrictFieldValidationOverridden(t *testing.T) { + calls := 0 + + fakeClient := testFieldValidationClient(t, metav1.FieldValidationWarn, func() { calls++ }) + wrappedClient := client.WithFieldValidation(fakeClient, metav1.FieldValidationStrict) + + ctx := t.Context() + dummyObj := &corev1.Namespace{} + + _ = wrappedClient.Create(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Update(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Patch(ctx, dummyObj, nil, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Status().Create(ctx, dummyObj, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Status().Update(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.Status().Patch(ctx, dummyObj, nil, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.SubResource("some-subresource").Create(ctx, dummyObj, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.SubResource("some-subresource").Update(ctx, dummyObj, client.FieldValidation(metav1.FieldValidationWarn)) + _ = wrappedClient.SubResource("some-subresource").Patch(ctx, dummyObj, nil, client.FieldValidation(metav1.FieldValidationWarn)) + + if expectedCalls := 9; calls != expectedCalls { + t.Fatalf("wrong number of calls to assertions: expected=%d; got=%d", expectedCalls, calls) + } +} + +// testFieldValidationClient is a helper function that checks if calls have the expected field validation, +// and calls the callback function on each intercepted call. +func testFieldValidationClient(t *testing.T, expectedFieldValidation string, callback func()) client.Client { + // TODO: we could use the dummyClient in interceptor pkg if we move it to an internal pkg + return fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + callback() + out := &client.CreateOptions{} + for _, f := range opts { + f.ApplyToCreate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsCreateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.CreateOptions{} + out.ApplyToCreate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + callback() + out := &client.UpdateOptions{} + for _, f := range opts { + f.ApplyToUpdate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsUpdateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.UpdateOptions{} + out.ApplyToUpdate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + Apply: func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + callback() + return nil + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + callback() + out := &client.PatchOptions{} + for _, f := range opts { + f.ApplyToPatch(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsPatchOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.PatchOptions{} + out.ApplyToPatch(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourceCreate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + callback() + out := &client.SubResourceCreateOptions{} + for _, f := range opts { + f.ApplyToSubResourceCreate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsCreateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.CreateOptions{} + out.ApplyToCreate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + callback() + out := &client.SubResourceUpdateOptions{} + for _, f := range opts { + f.ApplyToSubResourceUpdate(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsUpdateOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.UpdateOptions{} + out.ApplyToUpdate(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourcePatch: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + callback() + out := &client.SubResourcePatchOptions{} + for _, f := range opts { + f.ApplyToSubResourcePatch(out) + } + if got := out.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + if got := out.AsPatchOptions().FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + + co := &client.PatchOptions{} + out.ApplyToPatch(co) + if got := co.FieldValidation; expectedFieldValidation != got { + t.Fatalf("wrong field validation: expected=%q; got=%q", expectedFieldValidation, got) + } + return nil + }, + SubResourceApply: func(ctx context.Context, c client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + callback() + return nil + }, + }).Build() +} diff --git a/pkg/client/interceptor/intercept.go b/pkg/client/interceptor/intercept.go index 3d3f3cb011..b98af1a693 100644 --- a/pkg/client/interceptor/intercept.go +++ b/pkg/client/interceptor/intercept.go @@ -19,12 +19,14 @@ type Funcs struct { DeleteAllOf func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error Update func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error Patch func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error + Apply func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error Watch func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) SubResource func(client client.WithWatch, subResource string) client.SubResourceClient SubResourceGet func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error SubResourceCreate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error SubResourceUpdate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error SubResourcePatch func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error + SubResourceApply func(ctx context.Context, client client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error } // NewClient returns a new interceptor client that calls the functions in funcs instead of the underlying client's methods, if they are not nil. @@ -92,6 +94,14 @@ func (c interceptor) Patch(ctx context.Context, obj client.Object, patch client. return c.client.Patch(ctx, obj, patch, opts...) } +func (c interceptor) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + if c.funcs.Apply != nil { + return c.funcs.Apply(ctx, c.client, obj, opts...) + } + + return c.client.Apply(ctx, obj, opts...) +} + func (c interceptor) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { if c.funcs.DeleteAllOf != nil { return c.funcs.DeleteAllOf(ctx, c.client, obj, opts...) @@ -164,3 +174,10 @@ func (s subResourceInterceptor) Patch(ctx context.Context, obj client.Object, pa } return s.client.SubResource(s.subResourceName).Patch(ctx, obj, patch, opts...) } + +func (s subResourceInterceptor) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + if s.funcs.SubResourceApply != nil { + return s.funcs.SubResourceApply(ctx, s.client, s.subResourceName, obj, opts...) + } + return s.client.SubResource(s.subResourceName).Apply(ctx, obj, opts...) +} diff --git a/pkg/client/interceptor/intercept_test.go b/pkg/client/interceptor/intercept_test.go index a0536789b1..fb58dfeac1 100644 --- a/pkg/client/interceptor/intercept_test.go +++ b/pkg/client/interceptor/intercept_test.go @@ -16,8 +16,7 @@ import ( var _ = Describe("NewClient", func() { wrappedClient := dummyClient{} - ctx := context.Background() - It("should call the provided Get function", func() { + It("should call the provided Get function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { @@ -28,7 +27,7 @@ var _ = Describe("NewClient", func() { _ = client.Get(ctx, types.NamespacedName{}, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Get function is nil", func() { + It("should call the underlying client if the provided Get function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { @@ -40,7 +39,7 @@ var _ = Describe("NewClient", func() { _ = client2.Get(ctx, types.NamespacedName{}, nil) Expect(called).To(BeTrue()) }) - It("should call the provided List function", func() { + It("should call the provided List function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ List: func(ctx context.Context, client client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { @@ -51,7 +50,7 @@ var _ = Describe("NewClient", func() { _ = client.List(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided List function is nil", func() { + It("should call the underlying client if the provided List function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ List: func(ctx context.Context, client client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { @@ -63,7 +62,30 @@ var _ = Describe("NewClient", func() { _ = client2.List(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Create function", func() { + It("should call the provided Apply function", func(ctx SpecContext) { + var called bool + client := NewClient(wrappedClient, Funcs{ + Apply: func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + called = true + return nil + }, + }) + _ = client.Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Apply function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(wrappedClient, Funcs{ + Apply: func(ctx context.Context, client client.WithWatch, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + called = true + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the provided Create function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ Create: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.CreateOption) error { @@ -74,7 +96,7 @@ var _ = Describe("NewClient", func() { _ = client.Create(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Create function is nil", func() { + It("should call the underlying client if the provided Create function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ Create: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.CreateOption) error { @@ -86,7 +108,7 @@ var _ = Describe("NewClient", func() { _ = client2.Create(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Delete function", func() { + It("should call the provided Delete function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ Delete: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { @@ -97,7 +119,7 @@ var _ = Describe("NewClient", func() { _ = client.Delete(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Delete function is nil", func() { + It("should call the underlying client if the provided Delete function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ Delete: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { @@ -109,7 +131,7 @@ var _ = Describe("NewClient", func() { _ = client2.Delete(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the provided DeleteAllOf function", func() { + It("should call the provided DeleteAllOf function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ DeleteAllOf: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { @@ -120,7 +142,7 @@ var _ = Describe("NewClient", func() { _ = client.DeleteAllOf(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided DeleteAllOf function is nil", func() { + It("should call the underlying client if the provided DeleteAllOf function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ DeleteAllOf: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error { @@ -132,7 +154,7 @@ var _ = Describe("NewClient", func() { _ = client2.DeleteAllOf(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Update function", func() { + It("should call the provided Update function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ Update: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { @@ -143,7 +165,7 @@ var _ = Describe("NewClient", func() { _ = client.Update(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Update function is nil", func() { + It("should call the underlying client if the provided Update function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ Update: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { @@ -155,7 +177,7 @@ var _ = Describe("NewClient", func() { _ = client2.Update(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Patch function", func() { + It("should call the provided Patch function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ Patch: func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { @@ -166,7 +188,7 @@ var _ = Describe("NewClient", func() { _ = client.Patch(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Patch function is nil", func() { + It("should call the underlying client if the provided Patch function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ Patch: func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { @@ -178,7 +200,7 @@ var _ = Describe("NewClient", func() { _ = client2.Patch(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Watch function", func() { + It("should call the provided Watch function", func(ctx SpecContext) { var called bool client := NewClient(wrappedClient, Funcs{ Watch: func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { @@ -189,7 +211,7 @@ var _ = Describe("NewClient", func() { _, _ = client.Watch(ctx, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Watch function is nil", func() { + It("should call the underlying client if the provided Watch function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(wrappedClient, Funcs{ Watch: func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { @@ -229,8 +251,7 @@ var _ = Describe("NewClient", func() { var _ = Describe("NewSubResourceClient", func() { c := dummyClient{} - ctx := context.Background() - It("should call the provided Get function", func() { + It("should call the provided Get function", func(ctx SpecContext) { var called bool c := NewClient(c, Funcs{ SubResourceGet: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { @@ -242,7 +263,7 @@ var _ = Describe("NewSubResourceClient", func() { _ = c.SubResource("foo").Get(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Get function is nil", func() { + It("should call the underlying client if the provided Get function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(c, Funcs{ SubResourceGet: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { @@ -255,7 +276,7 @@ var _ = Describe("NewSubResourceClient", func() { _ = client2.SubResource("foo").Get(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Update function", func() { + It("should call the provided Update function", func(ctx SpecContext) { var called bool client := NewClient(c, Funcs{ SubResourceUpdate: func(_ context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { @@ -267,7 +288,7 @@ var _ = Describe("NewSubResourceClient", func() { _ = client.SubResource("foo").Update(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Update function is nil", func() { + It("should call the underlying client if the provided Update function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(c, Funcs{ SubResourceUpdate: func(_ context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { @@ -280,7 +301,7 @@ var _ = Describe("NewSubResourceClient", func() { _ = client2.SubResource("foo").Update(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Patch function", func() { + It("should call the provided Patch function", func(ctx SpecContext) { var called bool client := NewClient(c, Funcs{ SubResourcePatch: func(_ context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { @@ -292,7 +313,7 @@ var _ = Describe("NewSubResourceClient", func() { _ = client.SubResource("foo").Patch(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Patch function is nil", func() { + It("should call the underlying client if the provided Patch function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(c, Funcs{ SubResourcePatch: func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { @@ -305,7 +326,7 @@ var _ = Describe("NewSubResourceClient", func() { _ = client2.SubResource("foo").Patch(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the provided Create function", func() { + It("should call the provided Create function", func(ctx SpecContext) { var called bool client := NewClient(c, Funcs{ SubResourceCreate: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceCreateOption) error { @@ -317,7 +338,7 @@ var _ = Describe("NewSubResourceClient", func() { _ = client.SubResource("foo").Create(ctx, nil, nil) Expect(called).To(BeTrue()) }) - It("should call the underlying client if the provided Create function is nil", func() { + It("should call the underlying client if the provided Create function is nil", func(ctx SpecContext) { var called bool client1 := NewClient(c, Funcs{ SubResourceCreate: func(_ context.Context, client client.Client, subResourceName string, obj, subResource client.Object, opts ...client.SubResourceCreateOption) error { @@ -330,6 +351,31 @@ var _ = Describe("NewSubResourceClient", func() { _ = client2.SubResource("foo").Create(ctx, nil, nil) Expect(called).To(BeTrue()) }) + It("should call the provided Apply function", func(ctx SpecContext) { + var called bool + client := NewClient(c, Funcs{ + SubResourceApply: func(_ context.Context, client client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + _ = client.SubResource("foo").Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) + It("should call the underlying client if the provided Apply function is nil", func(ctx SpecContext) { + var called bool + client1 := NewClient(c, Funcs{ + SubResourceApply: func(_ context.Context, client client.Client, subResourceName string, obj runtime.ApplyConfiguration, opts ...client.SubResourceApplyOption) error { + called = true + Expect(subResourceName).To(BeEquivalentTo("foo")) + return nil + }, + }) + client2 := NewClient(client1, Funcs{}) + _ = client2.SubResource("foo").Apply(ctx, nil) + Expect(called).To(BeTrue()) + }) }) type dummyClient struct{} @@ -360,6 +406,10 @@ func (d dummyClient) Patch(ctx context.Context, obj client.Object, patch client. return nil } +func (d dummyClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...client.ApplyOption) error { + return nil +} + func (d dummyClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { return nil } diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 0ddda3163d..1af1f3a368 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -61,6 +61,9 @@ type Reader interface { // Writer knows how to create, delete, and update Kubernetes objects. type Writer interface { + // Apply applies the given apply configuration to the Kubernetes cluster. + Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error + // Create saves the object obj in the Kubernetes cluster. obj must be a // struct pointer so that obj can be updated with the content returned by the Server. Create(ctx context.Context, obj Object, opts ...CreateOption) error @@ -94,16 +97,16 @@ type SubResourceClientConstructor interface { // - ServiceAccount token creation: // sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} // token := &authenticationv1.TokenRequest{} - // c.SubResourceClient("token").Create(ctx, sa, token) + // c.SubResource("token").Create(ctx, sa, token) // // - Pod eviction creation: // pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} - // c.SubResourceClient("eviction").Create(ctx, pod, &policyv1.Eviction{}) + // c.SubResource("eviction").Create(ctx, pod, &policyv1.Eviction{}) // // - Pod binding creation: // pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} // binding := &corev1.Binding{Target: corev1.ObjectReference{Name: "my-node"}} - // c.SubResourceClient("binding").Create(ctx, pod, binding) + // c.SubResource("binding").Create(ctx, pod, binding) // // - CertificateSigningRequest approval: // csr := &certificatesv1.CertificateSigningRequest{ @@ -115,17 +118,17 @@ type SubResourceClientConstructor interface { // }}, // }, // } - // c.SubResourceClient("approval").Update(ctx, csr) + // c.SubResource("approval").Update(ctx, csr) // // - Scale retrieval: // dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} // scale := &autoscalingv1.Scale{} - // c.SubResourceClient("scale").Get(ctx, dep, scale) + // c.SubResource("scale").Get(ctx, dep, scale) // // - Scale update: // dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} // scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 2}} - // c.SubResourceClient("scale").Update(ctx, dep, client.WithSubResourceBody(scale)) + // c.SubResource("scale").Update(ctx, dep, client.WithSubResourceBody(scale)) SubResource(subResource string) SubResourceClient } @@ -142,6 +145,7 @@ type SubResourceWriter interface { // Create saves the subResource object in the Kubernetes cluster. obj must be a // struct pointer so that obj can be updated with the content returned by the Server. Create(ctx context.Context, obj Object, subResource Object, opts ...SubResourceCreateOption) error + // Update updates the fields corresponding to the status subresource for the // given obj. obj must be a struct pointer so that obj can be updated // with the content returned by the Server. @@ -151,6 +155,9 @@ type SubResourceWriter interface { // pointer so that obj can be updated with the content returned by the // Server. Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error + + // Apply applies the given apply configurations subresource. + Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error } // SubResourceClient knows how to perform CRU operations on Kubernetes objects. @@ -192,7 +199,7 @@ type IndexerFunc func(Object) []string // FieldIndexer knows how to index over a particular "field" such that it // can later be used by a field selector. type FieldIndexer interface { - // IndexFields adds an index with the given field name on the given object type + // IndexField adds an index with the given field name on the given object type // by using the given function to extract the value for that field. If you want // compatibility with the Kubernetes API server, only return one key, and only use // fields that the API server supports. Otherwise, you can return multiple keys, diff --git a/pkg/client/namespaced_client.go b/pkg/client/namespaced_client.go index 222dc79579..445e91b98b 100644 --- a/pkg/client/namespaced_client.go +++ b/pkg/client/namespaced_client.go @@ -19,10 +19,13 @@ package client import ( "context" "fmt" + "reflect" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) // NewNamespacedClient wraps an existing client enforcing the namespace value. @@ -147,6 +150,60 @@ func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, o return n.client.Patch(ctx, obj, patch, opts...) } +func (n *namespacedClient) setNamespaceForApplyConfigIfNamespaceScoped(obj runtime.ApplyConfiguration) error { + var gvk schema.GroupVersionKind + switch o := obj.(type) { + case applyConfiguration: + var err error + gvk, err = gvkFromApplyConfiguration(o) + if err != nil { + return err + } + case *unstructuredApplyConfiguration: + gvk = o.GroupVersionKind() + default: + return fmt.Errorf("object %T is not a valid apply configuration", obj) + } + isNamespaceScoped, err := apiutil.IsGVKNamespaced(gvk, n.RESTMapper()) + if err != nil { + return fmt.Errorf("error finding the scope of the object: %w", err) + } + if isNamespaceScoped { + switch o := obj.(type) { + case applyConfiguration: + if o.GetNamespace() != nil && *o.GetNamespace() != "" && *o.GetNamespace() != n.namespace { + return fmt.Errorf("namespace %s provided for the object %s does not match the namespace %s on the client", + *o.GetNamespace(), ptr.Deref(o.GetName(), ""), n.namespace) + } + v := reflect.ValueOf(o) + withNamespace := v.MethodByName("WithNamespace") + if !withNamespace.IsValid() { + return fmt.Errorf("ApplyConfiguration %T does not have a WithNamespace method", o) + } + if tp := withNamespace.Type(); tp.NumIn() != 1 || tp.In(0).Kind() != reflect.String { + return fmt.Errorf("WithNamespace method of ApplyConfiguration %T must take a single string argument", o) + } + withNamespace.Call([]reflect.Value{reflect.ValueOf(n.namespace)}) + case *unstructuredApplyConfiguration: + if o.GetNamespace() != "" && o.GetNamespace() != n.namespace { + return fmt.Errorf("namespace %s provided for the object %s does not match the namespace %s on the client", + o.GetNamespace(), o.GetName(), n.namespace) + } + o.SetNamespace(n.namespace) + } + } + + return nil +} + +func (n *namespacedClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + if err := n.setNamespaceForApplyConfigIfNamespaceScoped(obj); err != nil { + return err + } + + return n.client.Apply(ctx, obj, opts...) +} + // Get implements client.Client. func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { isNamespaceScoped, err := n.IsObjectNamespaced(obj) @@ -177,7 +234,10 @@ func (n *namespacedClient) Status() SubResourceWriter { // SubResource implements client.SubResourceClient. func (n *namespacedClient) SubResource(subResource string) SubResourceClient { - return &namespacedClientSubResourceClient{client: n.client.SubResource(subResource), namespace: n.namespace, namespacedclient: n} + return &namespacedClientSubResourceClient{ + client: n.client.SubResource(subResource), + namespacedclient: n, + } } // ensure namespacedClientSubResourceClient implements client.SubResourceClient. @@ -185,8 +245,7 @@ var _ SubResourceClient = &namespacedClientSubResourceClient{} type namespacedClientSubResourceClient struct { client SubResourceClient - namespace string - namespacedclient Client + namespacedclient *namespacedClient } func (nsw *namespacedClientSubResourceClient) Get(ctx context.Context, obj, subResource Object, opts ...SubResourceGetOption) error { @@ -196,12 +255,12 @@ func (nsw *namespacedClientSubResourceClient) Get(ctx context.Context, obj, subR } objectNamespace := obj.GetNamespace() - if objectNamespace != nsw.namespace && objectNamespace != "" { - return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) } if isNamespaceScoped && objectNamespace == "" { - obj.SetNamespace(nsw.namespace) + obj.SetNamespace(nsw.namespacedclient.namespace) } return nsw.client.Get(ctx, obj, subResource, opts...) @@ -214,12 +273,12 @@ func (nsw *namespacedClientSubResourceClient) Create(ctx context.Context, obj, s } objectNamespace := obj.GetNamespace() - if objectNamespace != nsw.namespace && objectNamespace != "" { - return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) } if isNamespaceScoped && objectNamespace == "" { - obj.SetNamespace(nsw.namespace) + obj.SetNamespace(nsw.namespacedclient.namespace) } return nsw.client.Create(ctx, obj, subResource, opts...) @@ -233,12 +292,12 @@ func (nsw *namespacedClientSubResourceClient) Update(ctx context.Context, obj Ob } objectNamespace := obj.GetNamespace() - if objectNamespace != nsw.namespace && objectNamespace != "" { - return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) } if isNamespaceScoped && objectNamespace == "" { - obj.SetNamespace(nsw.namespace) + obj.SetNamespace(nsw.namespacedclient.namespace) } return nsw.client.Update(ctx, obj, opts...) } @@ -251,12 +310,19 @@ func (nsw *namespacedClientSubResourceClient) Patch(ctx context.Context, obj Obj } objectNamespace := obj.GetNamespace() - if objectNamespace != nsw.namespace && objectNamespace != "" { - return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + if objectNamespace != nsw.namespacedclient.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespacedclient.namespace) } if isNamespaceScoped && objectNamespace == "" { - obj.SetNamespace(nsw.namespace) + obj.SetNamespace(nsw.namespacedclient.namespace) } return nsw.client.Patch(ctx, obj, patch, opts...) } + +func (nsw *namespacedClientSubResourceClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...SubResourceApplyOption) error { + if err := nsw.namespacedclient.setNamespaceForApplyConfigIfNamespaceScoped(obj); err != nil { + return err + } + return nsw.client.Apply(ctx, obj, opts...) +} diff --git a/pkg/client/namespaced_client_test.go b/pkg/client/namespaced_client_test.go index 6e1c2641a3..deae881d4a 100644 --- a/pkg/client/namespaced_client_test.go +++ b/pkg/client/namespaced_client_test.go @@ -25,21 +25,27 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - rbacv1 "k8s.io/api/rbac/v1" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + appsv1applyconfigurations "k8s.io/client-go/applyconfigurations/apps/v1" + corev1applyconfigurations "k8s.io/client-go/applyconfigurations/core/v1" + metav1applyconfigurations "k8s.io/client-go/applyconfigurations/meta/v1" + rbacv1applyconfigurations "k8s.io/client-go/applyconfigurations/rbac/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("NamespacedClient", func() { var dep *appsv1.Deployment + var acDep *appsv1applyconfigurations.DeploymentApplyConfiguration var ns = "default" - ctx := context.Background() var count uint64 = 0 var replicaCount int32 = 2 @@ -75,20 +81,36 @@ var _ = Describe("NamespacedClient", func() { }, }, } + acDep = appsv1applyconfigurations.Deployment(dep.Name, ""). + WithLabels(dep.Labels). + WithSpec(appsv1applyconfigurations.DeploymentSpec(). + WithReplicas(*dep.Spec.Replicas). + WithSelector(metav1applyconfigurations.LabelSelector().WithMatchLabels(dep.Spec.Selector.MatchLabels)). + WithTemplate(corev1applyconfigurations.PodTemplateSpec(). + WithLabels(dep.Spec.Template.Labels). + WithSpec(corev1applyconfigurations.PodSpec(). + WithContainers(corev1applyconfigurations.Container(). + WithName(dep.Spec.Template.Spec.Containers[0].Name). + WithImage(dep.Spec.Template.Spec.Containers[0].Image), + ), + ), + ), + ) + }) Describe("Get", func() { - - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { var err error dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully Get a namespace-scoped object", func() { + + It("should successfully Get a namespace-scoped object", func(ctx SpecContext) { name := types.NamespacedName{Name: dep.Name} result := &appsv1.Deployment{} @@ -97,7 +119,7 @@ var _ = Describe("NamespacedClient", func() { }) It("should error when namespace provided in the object is different than the one "+ - "specified in client", func() { + "specified in client", func(ctx SpecContext) { name := types.NamespacedName{Name: dep.Name, Namespace: "non-default"} result := &appsv1.Deployment{} @@ -106,17 +128,17 @@ var _ = Describe("NamespacedClient", func() { }) Describe("List", func() { - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { var err error dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully List objects when namespace is not specified with the object", func() { + It("should successfully List objects when namespace is not specified with the object", func(ctx SpecContext) { result := &appsv1.DeploymentList{} opts := client.MatchingLabels(dep.Labels) @@ -125,7 +147,7 @@ var _ = Describe("NamespacedClient", func() { Expect(result.Items[0]).To(BeEquivalentTo(*dep)) }) - It("should List objects from the namespace specified in the client", func() { + It("should List objects from the namespace specified in the client", func(ctx SpecContext) { result := &appsv1.DeploymentList{} opts := client.InNamespace("non-default") @@ -135,12 +157,72 @@ var _ = Describe("NamespacedClient", func() { }) }) + Describe("Apply", func() { + AfterEach(func(ctx SpecContext) { + deleteDeployment(ctx, dep, ns) + }) + + It("should successfully apply an object in the right namespace", func(ctx SpecContext) { + err := getClient().Apply(ctx, acDep, client.FieldOwner("test")) + Expect(err).NotTo(HaveOccurred()) + + res, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.GetNamespace()).To(BeEquivalentTo(ns)) + }) + + It("should successfully apply an object in the right namespace through unstructured", func(ctx SpecContext) { + serialized, err := json.Marshal(acDep) + Expect(err).NotTo(HaveOccurred()) + u := &unstructured.Unstructured{} + Expect(json.Unmarshal(serialized, &u.Object)).To(Succeed()) + err = getClient().Apply(ctx, client.ApplyConfigurationFromUnstructured(u), client.FieldOwner("test")) + Expect(err).NotTo(HaveOccurred()) + + res, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.GetNamespace()).To(BeEquivalentTo(ns)) + }) + + It("should not create an object if the namespace of the object is different", func(ctx SpecContext) { + acDep.WithNamespace("non-default") + err := getClient().Apply(ctx, acDep, client.FieldOwner("test")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not match the namespace")) + }) + + It("should not create an object through unstructured if the namespace of the object is different", func(ctx SpecContext) { + acDep.WithNamespace("non-default") + serialized, err := json.Marshal(acDep) + Expect(err).NotTo(HaveOccurred()) + u := &unstructured.Unstructured{} + Expect(json.Unmarshal(serialized, &u.Object)).To(Succeed()) + err = getClient().Apply(ctx, client.ApplyConfigurationFromUnstructured(u), client.FieldOwner("test")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not match the namespace")) + }) + + It("should create a cluster scoped object", func(ctx SpecContext) { + cr := rbacv1applyconfigurations.ClusterRole(fmt.Sprintf("clusterRole-%v", count)) + + err := getClient().Apply(ctx, cr, client.FieldOwner("test")) + Expect(err).NotTo(HaveOccurred()) + + By("checking if the object was created") + res, err := clientset.RbacV1().ClusterRoles().Get(ctx, *cr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(res).NotTo(BeNil()) + + deleteClusterRole(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: *cr.Name}}) + }) + }) + Describe("Create", func() { - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully create object in the right namespace", func() { + It("should successfully create object in the right namespace", func(ctx SpecContext) { By("creating the object initially") err := getClient().Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) @@ -151,13 +233,13 @@ var _ = Describe("NamespacedClient", func() { Expect(res.GetNamespace()).To(BeEquivalentTo(ns)) }) - It("should not create object if the namespace of the object is different", func() { + It("should not create object if the namespace of the object is different", func(ctx SpecContext) { By("creating the object initially") dep.SetNamespace("non-default") err := getClient().Create(ctx, dep) Expect(err).To(HaveOccurred()) }) - It("should create a cluster scoped object", func() { + It("should create a cluster scoped object", func(ctx SpecContext) { cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("clusterRole-%v", count), @@ -186,16 +268,16 @@ var _ = Describe("NamespacedClient", func() { Describe("Update", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) dep.Annotations = map[string]string{"foo": "bar"} Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully update the provided object", func() { + It("should successfully update the provided object", func(ctx SpecContext) { By("updating the Deployment") err = getClient().Update(ctx, dep) Expect(err).NotTo(HaveOccurred()) @@ -208,7 +290,7 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should successfully update the provided object when namespace is not provided", func() { + It("should successfully update the provided object when namespace is not provided", func(ctx SpecContext) { By("updating the Deployment") dep.SetNamespace("") err = getClient().Update(ctx, dep) @@ -222,14 +304,14 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Annotations["foo"]).To(Equal("bar")) }) - It("should not update when object namespace is different", func() { + It("should not update when object namespace is different", func(ctx SpecContext) { By("updating the Deployment") dep.SetNamespace("non-default") err = getClient().Update(ctx, dep) Expect(err).To(HaveOccurred()) }) - It("should not update any object from other namespace", func() { + It("should not update any object from other namespace", func(ctx SpecContext) { By("creating a new namespace") tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "non-default-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) @@ -266,7 +348,7 @@ var _ = Describe("NamespacedClient", func() { deleteNamespace(ctx, tns) }) - It("should update a cluster scoped resource", func() { + It("should update a cluster scoped resource", func(ctx SpecContext) { changedCR := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("clusterRole-%v", count), @@ -302,16 +384,16 @@ var _ = Describe("NamespacedClient", func() { Describe("Patch", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully modify the object using Patch", func() { + It("should successfully modify the object using Patch", func(ctx SpecContext) { By("Applying Patch") err = getClient().Patch(ctx, dep, client.RawPatch(types.MergePatchType, generatePatch())) Expect(err).NotTo(HaveOccurred()) @@ -323,7 +405,7 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.GetNamespace()).To(Equal(ns)) }) - It("should successfully modify the object using Patch when namespace is not provided", func() { + It("should successfully modify the object using Patch when namespace is not provided", func(ctx SpecContext) { By("Applying Patch") dep.SetNamespace("") err = getClient().Patch(ctx, dep, client.RawPatch(types.MergePatchType, generatePatch())) @@ -336,13 +418,13 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.GetNamespace()).To(Equal(ns)) }) - It("should not modify the object when namespace of the object is different", func() { + It("should not modify the object when namespace of the object is different", func(ctx SpecContext) { dep.SetNamespace("non-default") err = getClient().Patch(ctx, dep, client.RawPatch(types.MergePatchType, generatePatch())) Expect(err).To(HaveOccurred()) }) - It("should not modify an object from a different namespace", func() { + It("should not modify an object from a different namespace", func(ctx SpecContext) { By("creating a new namespace") tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "non-default-2"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) @@ -377,7 +459,7 @@ var _ = Describe("NamespacedClient", func() { deleteNamespace(ctx, tns) }) - It("should successfully modify cluster scoped resource", func() { + It("should successfully modify cluster scoped resource", func(ctx SpecContext) { cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("clusterRole-%v", count), @@ -411,15 +493,15 @@ var _ = Describe("NamespacedClient", func() { Describe("Delete and DeleteAllOf", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should successfully delete an object when namespace is not specified", func() { + It("should successfully delete an object when namespace is not specified", func(ctx SpecContext) { By("deleting the object") dep.SetNamespace("") err = getClient().Delete(ctx, dep) @@ -430,7 +512,7 @@ var _ = Describe("NamespacedClient", func() { Expect(err).To(HaveOccurred()) }) - It("should successfully delete all of the deployments in the given namespace", func() { + It("should successfully delete all of the deployments in the given namespace", func(ctx SpecContext) { By("Deleting all objects in the namespace") err = getClient().DeleteAllOf(ctx, dep) Expect(err).NotTo(HaveOccurred()) @@ -440,7 +522,7 @@ var _ = Describe("NamespacedClient", func() { Expect(err).To(HaveOccurred()) }) - It("should not delete deployments in other namespaces", func() { + It("should not delete deployments in other namespaces", func(ctx SpecContext) { tns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "non-default-3"}} _, err = clientset.CoreV1().Namespaces().Create(ctx, tns, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -482,16 +564,16 @@ var _ = Describe("NamespacedClient", func() { Describe("SubResourceWriter", func() { var err error - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) - It("should change objects via update status", func() { + It("should change objects via update status", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 @@ -504,7 +586,7 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Status.Replicas).To(BeEquivalentTo(99)) }) - It("should not change objects via update status when object namespace is different", func() { + It("should not change objects via update status when object namespace is different", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.SetNamespace("test") changedDep.Status.Replicas = 99 @@ -512,7 +594,7 @@ var _ = Describe("NamespacedClient", func() { Expect(getClient().SubResource("status").Update(ctx, changedDep)).To(HaveOccurred()) }) - It("should change objects via status patch", func() { + It("should change objects via status patch", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 @@ -525,17 +607,59 @@ var _ = Describe("NamespacedClient", func() { Expect(actual.Status.Replicas).To(BeEquivalentTo(99)) }) - It("should not change objects via status patch when object namespace is different", func() { + It("should not change objects via status patch when object namespace is different", func(ctx SpecContext) { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 changedDep.SetNamespace("test") Expect(getClient().SubResource("status").Patch(ctx, changedDep, client.MergeFrom(dep))).To(HaveOccurred()) }) + + It("should change objects via status apply", func(ctx SpecContext) { + deploymentAC, err := appsv1applyconfigurations.ExtractDeployment(dep, "test-owner") + Expect(err).NotTo(HaveOccurred()) + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(99)), + }) + + Expect(getClient().SubResource("status").Apply(ctx, deploymentAC, client.FieldOwner("test-owner"))).To(Succeed()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual.GetNamespace()).To(BeEquivalentTo(ns)) + Expect(actual.Status.Replicas).To(BeEquivalentTo(99)) + }) + + It("should set namespace on ApplyConfiguration when applying via SubResource", func(ctx SpecContext) { + deploymentAC := appsv1applyconfigurations.Deployment(dep.Name, "") + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(50)), + }) + + Expect(getClient().SubResource("status").Apply(ctx, deploymentAC, client.FieldOwner("test-owner"))).To(Succeed()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).NotTo(BeNil()) + Expect(actual.GetNamespace()).To(BeEquivalentTo(ns)) + Expect(actual.Status.Replicas).To(BeEquivalentTo(50)) + }) + + It("should fail when applying via SubResource with conflicting namespace", func(ctx SpecContext) { + deploymentAC := appsv1applyconfigurations.Deployment(dep.Name, "different-namespace") + deploymentAC.WithStatus(&appsv1applyconfigurations.DeploymentStatusApplyConfiguration{ + Replicas: ptr.To(int32(25)), + }) + + err := getClient().SubResource("status").Apply(ctx, deploymentAC, client.FieldOwner("test-owner")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("namespace")) + }) }) Describe("Test on invalid objects", func() { - It("should refuse to perform operations on invalid object", func() { + It("should refuse to perform operations on invalid object", func(ctx SpecContext) { err := getClient().Create(ctx, nil) Expect(err).To(HaveOccurred()) diff --git a/pkg/client/options.go b/pkg/client/options.go index d81bf25de9..a6b921171a 100644 --- a/pkg/client/options.go +++ b/pkg/client/options.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "k8s.io/utils/ptr" ) // {{{ "Functional" Option Interfaces @@ -61,6 +62,12 @@ type PatchOption interface { ApplyToPatch(*PatchOptions) } +// ApplyOption is some configuration that modifies options for an apply request. +type ApplyOption interface { + // ApplyToApply applies this configuration to the given apply options. + ApplyToApply(*ApplyOptions) +} + // DeleteAllOfOption is some configuration that modifies options for a delete request. type DeleteAllOfOption interface { // ApplyToDeleteAllOf applies this configuration to the given deletecollection options. @@ -90,6 +97,12 @@ type SubResourcePatchOption interface { ApplyToSubResourcePatch(*SubResourcePatchOptions) } +// SubResourceApplyOption configures a subresource apply request. +type SubResourceApplyOption interface { + // ApplyToSubResourceApply applies the configuration on the given patch options. + ApplyToSubResourceApply(*SubResourceApplyOptions) +} + // }}} // {{{ Multi-Type Options @@ -115,7 +128,12 @@ func (dryRunAll) ApplyToPatch(opts *PatchOptions) { opts.DryRun = []string{metav1.DryRunAll} } -// ApplyToPatch applies this configuration to the given delete options. +// ApplyToApply applies this configuration to the given apply options. +func (dryRunAll) ApplyToApply(opts *ApplyOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + +// ApplyToDelete applies this configuration to the given delete options. func (dryRunAll) ApplyToDelete(opts *DeleteOptions) { opts.DryRun = []string{metav1.DryRunAll} } @@ -136,6 +154,10 @@ func (dryRunAll) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { opts.DryRun = []string{metav1.DryRunAll} } +func (dryRunAll) ApplyToSubResourceApply(opts *SubResourceApplyOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + // FieldOwner set the field manager name for the given server-side apply patch. type FieldOwner string @@ -154,6 +176,11 @@ func (f FieldOwner) ApplyToUpdate(opts *UpdateOptions) { opts.FieldManager = string(f) } +// ApplyToApply applies this configuration to the given apply options. +func (f FieldOwner) ApplyToApply(opts *ApplyOptions) { + opts.FieldManager = string(f) +} + // ApplyToSubResourcePatch applies this configuration to the given patch options. func (f FieldOwner) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { opts.FieldManager = string(f) @@ -169,6 +196,44 @@ func (f FieldOwner) ApplyToSubResourceUpdate(opts *SubResourceUpdateOptions) { opts.FieldManager = string(f) } +// ApplyToSubResourceApply applies this configuration to the given apply options. +func (f FieldOwner) ApplyToSubResourceApply(opts *SubResourceApplyOptions) { + opts.FieldManager = string(f) +} + +// FieldValidation configures field validation for the given requests. +type FieldValidation string + +// ApplyToPatch applies this configuration to the given patch options. +func (f FieldValidation) ApplyToPatch(opts *PatchOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToCreate applies this configuration to the given create options. +func (f FieldValidation) ApplyToCreate(opts *CreateOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToUpdate applies this configuration to the given update options. +func (f FieldValidation) ApplyToUpdate(opts *UpdateOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToSubResourcePatch applies this configuration to the given patch options. +func (f FieldValidation) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToSubResourceCreate applies this configuration to the given create options. +func (f FieldValidation) ApplyToSubResourceCreate(opts *SubResourceCreateOptions) { + opts.FieldValidation = string(f) +} + +// ApplyToSubResourceUpdate applies this configuration to the given update options. +func (f FieldValidation) ApplyToSubResourceUpdate(opts *SubResourceUpdateOptions) { + opts.FieldValidation = string(f) +} + // }}} // {{{ Create Options @@ -187,6 +252,24 @@ type CreateOptions struct { // this request. It must be set with server-side apply. FieldManager string + // fieldValidation instructs the server on how to handle + // objects in the request (POST/PUT/PATCH) containing unknown + // or duplicate fields. Valid values are: + // - Ignore: This will ignore any unknown fields that are silently + // dropped from the object, and will ignore all but the last duplicate + // field that the decoder encounters. This is the default behavior + // prior to v1.23. + // - Warn: This will send a warning via the standard warning response + // header for each unknown field that is dropped from the object, and + // for each duplicate field that is encountered. The request will + // still succeed if there are no other errors, and will only persist + // the last of any duplicate fields. This is the default in v1.23+ + // - Strict: This will fail the request with a BadRequest error if + // any unknown fields would be dropped from the object, or if any + // duplicate fields are present. The error returned from the server + // will contain all unknown and duplicate fields encountered. + FieldValidation string + // Raw represents raw CreateOptions, as passed to the API server. Raw *metav1.CreateOptions } @@ -203,6 +286,7 @@ func (o *CreateOptions) AsCreateOptions() *metav1.CreateOptions { o.Raw.DryRun = o.DryRun o.Raw.FieldManager = o.FieldManager + o.Raw.FieldValidation = o.FieldValidation return o.Raw } @@ -223,6 +307,9 @@ func (o *CreateOptions) ApplyToCreate(co *CreateOptions) { if o.FieldManager != "" { co.FieldManager = o.FieldManager } + if o.FieldValidation != "" { + co.FieldValidation = o.FieldValidation + } if o.Raw != nil { co.Raw = o.Raw } @@ -376,6 +463,12 @@ type GetOptions struct { // Raw represents raw GetOptions, as passed to the API server. Note // that these may not be respected by all implementations of interface. Raw *metav1.GetOptions + + // UnsafeDisableDeepCopy indicates not to deep copy objects during get object. + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + // +optional + UnsafeDisableDeepCopy *bool } var _ GetOption = &GetOptions{} @@ -385,6 +478,9 @@ func (o *GetOptions) ApplyToGet(lo *GetOptions) { if o.Raw != nil { lo.Raw = o.Raw } + if o.UnsafeDisableDeepCopy != nil { + lo.UnsafeDisableDeepCopy = o.UnsafeDisableDeepCopy + } } // AsGetOptions returns these options as a flattened metav1.GetOptions. @@ -419,7 +515,7 @@ type ListOptions struct { LabelSelector labels.Selector // FieldSelector filters results by a particular field. In order // to use this with cache-based implementations, restrict usage to - // a single field-value pair that's been added to the indexers. + // exact match field-value pair that's been added to the indexers. FieldSelector fields.Selector // Namespace represents the namespace to list for, or empty for @@ -514,7 +610,8 @@ type MatchingLabels map[string]string func (m MatchingLabels) ApplyToList(opts *ListOptions) { // TODO(directxman12): can we avoid reserializing this over and over? if opts.LabelSelector == nil { - opts.LabelSelector = labels.NewSelector() + opts.LabelSelector = labels.SelectorFromValidatedSet(map[string]string(m)) + return } // If there's already a selector, we need to AND the two together. noValidSel := labels.SelectorFromValidatedSet(map[string]string(m)) @@ -562,6 +659,9 @@ type MatchingLabelsSelector struct { // ApplyToList applies this configuration to the given list options. func (m MatchingLabelsSelector) ApplyToList(opts *ListOptions) { + if m.Selector == nil { + m.Selector = labels.Nothing() + } opts.LabelSelector = m } @@ -595,6 +695,9 @@ type MatchingFieldsSelector struct { // ApplyToList applies this configuration to the given list options. func (m MatchingFieldsSelector) ApplyToList(opts *ListOptions) { + if m.Selector == nil { + m.Selector = fields.Nothing() + } opts.FieldSelector = m } @@ -636,15 +739,14 @@ func (l Limit) ApplyToList(opts *ListOptions) { // otherwise you will mutate the object in the cache. type UnsafeDisableDeepCopyOption bool +// ApplyToGet applies this configuration to the given an Get options. +func (d UnsafeDisableDeepCopyOption) ApplyToGet(opts *GetOptions) { + opts.UnsafeDisableDeepCopy = ptr.To(bool(d)) +} + // ApplyToList applies this configuration to the given an List options. func (d UnsafeDisableDeepCopyOption) ApplyToList(opts *ListOptions) { - definitelyTrue := true - definitelyFalse := false - if d { - opts.UnsafeDisableDeepCopy = &definitelyTrue - } else { - opts.UnsafeDisableDeepCopy = &definitelyFalse - } + opts.UnsafeDisableDeepCopy = ptr.To(bool(d)) } // UnsafeDisableDeepCopy indicates not to deep copy objects during list objects. @@ -678,6 +780,24 @@ type UpdateOptions struct { // this request. It must be set with server-side apply. FieldManager string + // fieldValidation instructs the server on how to handle + // objects in the request (POST/PUT/PATCH) containing unknown + // or duplicate fields. Valid values are: + // - Ignore: This will ignore any unknown fields that are silently + // dropped from the object, and will ignore all but the last duplicate + // field that the decoder encounters. This is the default behavior + // prior to v1.23. + // - Warn: This will send a warning via the standard warning response + // header for each unknown field that is dropped from the object, and + // for each duplicate field that is encountered. The request will + // still succeed if there are no other errors, and will only persist + // the last of any duplicate fields. This is the default in v1.23+ + // - Strict: This will fail the request with a BadRequest error if + // any unknown fields would be dropped from the object, or if any + // duplicate fields are present. The error returned from the server + // will contain all unknown and duplicate fields encountered. + FieldValidation string + // Raw represents raw UpdateOptions, as passed to the API server. Raw *metav1.UpdateOptions } @@ -694,6 +814,7 @@ func (o *UpdateOptions) AsUpdateOptions() *metav1.UpdateOptions { o.Raw.DryRun = o.DryRun o.Raw.FieldManager = o.FieldManager + o.Raw.FieldValidation = o.FieldValidation return o.Raw } @@ -716,6 +837,9 @@ func (o *UpdateOptions) ApplyToUpdate(uo *UpdateOptions) { if o.FieldManager != "" { uo.FieldManager = o.FieldManager } + if o.FieldValidation != "" { + uo.FieldValidation = o.FieldValidation + } if o.Raw != nil { uo.Raw = o.Raw } @@ -744,6 +868,24 @@ type PatchOptions struct { // this request. It must be set with server-side apply. FieldManager string + // fieldValidation instructs the server on how to handle + // objects in the request (POST/PUT/PATCH) containing unknown + // or duplicate fields. Valid values are: + // - Ignore: This will ignore any unknown fields that are silently + // dropped from the object, and will ignore all but the last duplicate + // field that the decoder encounters. This is the default behavior + // prior to v1.23. + // - Warn: This will send a warning via the standard warning response + // header for each unknown field that is dropped from the object, and + // for each duplicate field that is encountered. The request will + // still succeed if there are no other errors, and will only persist + // the last of any duplicate fields. This is the default in v1.23+ + // - Strict: This will fail the request with a BadRequest error if + // any unknown fields would be dropped from the object, or if any + // duplicate fields are present. The error returned from the server + // will contain all unknown and duplicate fields encountered. + FieldValidation string + // Raw represents raw PatchOptions, as passed to the API server. Raw *metav1.PatchOptions } @@ -767,9 +909,18 @@ func (o *PatchOptions) AsPatchOptions() *metav1.PatchOptions { o.Raw = &metav1.PatchOptions{} } - o.Raw.DryRun = o.DryRun - o.Raw.Force = o.Force - o.Raw.FieldManager = o.FieldManager + if o.DryRun != nil { + o.Raw.DryRun = o.DryRun + } + if o.Force != nil { + o.Raw.Force = o.Force + } + if o.FieldManager != "" { + o.Raw.FieldManager = o.FieldManager + } + if o.FieldValidation != "" { + o.Raw.FieldValidation = o.FieldValidation + } return o.Raw } @@ -786,6 +937,9 @@ func (o *PatchOptions) ApplyToPatch(po *PatchOptions) { if o.FieldManager != "" { po.FieldManager = o.FieldManager } + if o.FieldValidation != "" { + po.FieldValidation = o.FieldValidation + } if o.Raw != nil { po.Raw = o.Raw } @@ -799,13 +953,19 @@ var ForceOwnership = forceOwnership{} type forceOwnership struct{} func (forceOwnership) ApplyToPatch(opts *PatchOptions) { - definitelyTrue := true - opts.Force = &definitelyTrue + opts.Force = ptr.To(true) } func (forceOwnership) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { - definitelyTrue := true - opts.Force = &definitelyTrue + opts.Force = ptr.To(true) +} + +func (forceOwnership) ApplyToApply(opts *ApplyOptions) { + opts.Force = ptr.To(true) +} + +func (forceOwnership) ApplyToSubResourceApply(opts *SubResourceApplyOptions) { + opts.Force = ptr.To(true) } // }}} @@ -839,3 +999,57 @@ func (o *DeleteAllOfOptions) ApplyToDeleteAllOf(do *DeleteAllOfOptions) { } // }}} + +// ApplyOptions are the options for an apply request. +type ApplyOptions struct { + // When present, indicates that modifications should not be + // persisted. An invalid or unrecognized dryRun directive will + // result in an error response and no further processing of the + // request. Valid values are: + // - All: all dry run stages will be processed + DryRun []string + + // Force is going to "force" Apply requests. It means user will + // re-acquire conflicting fields owned by other people. + Force *bool + + // fieldManager is a name associated with the actor or entity + // that is making these changes. The value must be less than or + // 128 characters long, and only contain printable characters, + // as defined by https://golang.org/pkg/unicode/#IsPrint. This + // field is required. + // + // +required + FieldManager string +} + +// ApplyOptions applies the given opts onto the ApplyOptions +func (o *ApplyOptions) ApplyOptions(opts []ApplyOption) *ApplyOptions { + for _, opt := range opts { + opt.ApplyToApply(o) + } + return o +} + +// ApplyToApply applies the given opts onto the ApplyOptions +func (o *ApplyOptions) ApplyToApply(opts *ApplyOptions) { + if o.DryRun != nil { + opts.DryRun = o.DryRun + } + if o.Force != nil { + opts.Force = o.Force + } + + if o.FieldManager != "" { + opts.FieldManager = o.FieldManager + } +} + +// AsPatchOptions constructs patch options from the given ApplyOptions +func (o *ApplyOptions) AsPatchOptions() *metav1.PatchOptions { + return &metav1.PatchOptions{ + DryRun: o.DryRun, + Force: o.Force, + FieldManager: o.FieldManager, + } +} diff --git a/pkg/client/options_test.go b/pkg/client/options_test.go index 8a90c14dda..082586bca3 100644 --- a/pkg/client/options_test.go +++ b/pkg/client/options_test.go @@ -23,7 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" - utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -36,12 +36,38 @@ var _ = Describe("ListOptions", func() { o.ApplyToList(newListOpts) Expect(newListOpts).To(Equal(o)) }) + It("Should set LabelSelector with MatchingLabelsSelector", func() { + labelSelector, err := labels.Parse("a=b") + Expect(err).NotTo(HaveOccurred()) + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingLabelsSelector{Selector: labelSelector}}) + expectedListOpts := &client.ListOptions{LabelSelector: client.MatchingLabelsSelector{Selector: labelSelector}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) + It("Should set LabelSelector to nothing with empty MatchingLabelsSelector", func() { + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingLabelsSelector{}}) + expectedListOpts := &client.ListOptions{LabelSelector: client.MatchingLabelsSelector{Selector: labels.Nothing()}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) It("Should set FieldSelector", func() { o := &client.ListOptions{FieldSelector: fields.Nothing()} newListOpts := &client.ListOptions{} o.ApplyToList(newListOpts) Expect(newListOpts).To(Equal(o)) }) + It("Should set FieldSelector with MatchingFieldsSelector", func() { + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingFieldsSelector{Selector: fields.Nothing()}}) + expectedListOpts := &client.ListOptions{FieldSelector: client.MatchingFieldsSelector{Selector: fields.Nothing()}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) + It("Should set FieldSelector to nothing with empty MatchingFieldsSelector", func() { + newListOpts := &client.ListOptions{} + newListOpts.ApplyOptions([]client.ListOption{client.MatchingFieldsSelector{}}) + expectedListOpts := &client.ListOptions{FieldSelector: client.MatchingFieldsSelector{Selector: fields.Nothing()}} + Expect(newListOpts).To(Equal(expectedListOpts)) + }) It("Should set Namespace", func() { o := &client.ListOptions{Namespace: "my-ns"} newListOpts := &client.ListOptions{} @@ -66,6 +92,19 @@ var _ = Describe("ListOptions", func() { o.ApplyToList(newListOpts) Expect(newListOpts).To(Equal(o)) }) + It("Should set UnsafeDisableDeepCopy", func() { + definitelyTrue := true + o := &client.ListOptions{UnsafeDisableDeepCopy: &definitelyTrue} + newListOpts := &client.ListOptions{} + o.ApplyToList(newListOpts) + Expect(newListOpts).To(Equal(o)) + }) + It("Should set UnsafeDisableDeepCopy through option", func() { + listOpts := &client.ListOptions{} + client.UnsafeDisableDeepCopy.ApplyToList(listOpts) + Expect(listOpts.UnsafeDisableDeepCopy).ToNot(BeNil()) + Expect(*listOpts.UnsafeDisableDeepCopy).To(BeTrue()) + }) It("Should not set anything", func() { o := &client.ListOptions{} newListOpts := &client.ListOptions{} @@ -81,6 +120,40 @@ var _ = Describe("GetOptions", func() { o.ApplyToGet(newGetOpts) Expect(newGetOpts).To(Equal(o)) }) + It("Should set UnsafeDisableDeepCopy", func() { + definitelyTrue := true + o := &client.GetOptions{UnsafeDisableDeepCopy: &definitelyTrue} + newGetOpts := &client.GetOptions{} + o.ApplyToGet(newGetOpts) + Expect(newGetOpts).To(Equal(o)) + }) + It("Should set UnsafeDisableDeepCopy through option", func() { + getOpts := &client.GetOptions{} + client.UnsafeDisableDeepCopy.ApplyToGet(getOpts) + Expect(getOpts.UnsafeDisableDeepCopy).ToNot(BeNil()) + Expect(*getOpts.UnsafeDisableDeepCopy).To(BeTrue()) + }) +}) + +var _ = Describe("ApplyOptions", func() { + It("Should set DryRun", func() { + o := &client.ApplyOptions{DryRun: []string{"Hello", "Theodore"}} + newApplyOpts := &client.ApplyOptions{} + o.ApplyToApply(newApplyOpts) + Expect(newApplyOpts).To(Equal(o)) + }) + It("Should set Force", func() { + o := &client.ApplyOptions{Force: ptr.To(true)} + newApplyOpts := &client.ApplyOptions{} + o.ApplyToApply(newApplyOpts) + Expect(newApplyOpts).To(Equal(o)) + }) + It("Should set FieldManager", func() { + o := &client.ApplyOptions{FieldManager: "field-manager"} + newApplyOpts := &client.ApplyOptions{} + o.ApplyToApply(newApplyOpts) + Expect(newApplyOpts).To(Equal(o)) + }) }) var _ = Describe("CreateOptions", func() { @@ -112,7 +185,7 @@ var _ = Describe("CreateOptions", func() { var _ = Describe("DeleteOptions", func() { It("Should set GracePeriodSeconds", func() { - o := &client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64(42)} + o := &client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(42))} newDeleteOpts := &client.DeleteOptions{} o.ApplyToDelete(newDeleteOpts) Expect(newDeleteOpts).To(Equal(o)) @@ -185,7 +258,7 @@ var _ = Describe("PatchOptions", func() { Expect(newPatchOpts).To(Equal(o)) }) It("Should set Force", func() { - o := &client.PatchOptions{Force: utilpointer.Bool(true)} + o := &client.PatchOptions{Force: ptr.To(true)} newPatchOpts := &client.PatchOptions{} o.ApplyToPatch(newPatchOpts) Expect(newPatchOpts).To(Equal(o)) @@ -218,7 +291,7 @@ var _ = Describe("DeleteAllOfOptions", func() { Expect(newDeleteAllOfOpts).To(Equal(o)) }) It("Should set DeleleteOptions", func() { - o := &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64(44)}} + o := &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(44))}} newDeleteAllOfOpts := &client.DeleteAllOfOptions{} o.ApplyToDeleteAllOf(newDeleteAllOfOpts) Expect(newDeleteAllOfOpts).To(Equal(o)) @@ -252,6 +325,63 @@ var _ = Describe("MatchingLabels", func() { }) }) +var _ = Describe("DryRunAll", func() { + It("Should apply to ApplyOptions", func() { + o := &client.ApplyOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToApply(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to CreateOptions", func() { + o := &client.CreateOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToCreate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to UpdateOptions", func() { + o := &client.UpdateOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToUpdate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to PatchOptions", func() { + o := &client.PatchOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToPatch(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to DeleteOptions", func() { + o := &client.DeleteOptions{DryRun: []string{"server"}} + t := client.DryRunAll + t.ApplyToDelete(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourcePatchOptions", func() { + o := &client.SubResourcePatchOptions{PatchOptions: client.PatchOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourcePatch(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourceCreateOptions", func() { + o := &client.SubResourceCreateOptions{CreateOptions: client.CreateOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourceCreate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourceUpdateOptions", func() { + o := &client.SubResourceUpdateOptions{UpdateOptions: client.UpdateOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourceUpdate(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) + It("Should apply to SubResourceApplyOptions", func() { + o := &client.SubResourceApplyOptions{ApplyOptions: client.ApplyOptions{DryRun: []string{"server"}}} + t := client.DryRunAll + t.ApplyToSubResourceApply(o) + Expect(o.DryRun).To(Equal([]string{metav1.DryRunAll})) + }) +}) + var _ = Describe("FieldOwner", func() { It("Should apply to PatchOptions", func() { o := &client.PatchOptions{FieldManager: "bar"} @@ -259,6 +389,12 @@ var _ = Describe("FieldOwner", func() { t.ApplyToPatch(o) Expect(o.FieldManager).To(Equal("foo")) }) + It("Should apply to ApplyOptions", func() { + o := &client.ApplyOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToApply(o) + Expect(o.FieldManager).To(Equal("foo")) + }) It("Should apply to CreateOptions", func() { o := &client.CreateOptions{FieldManager: "bar"} t := client.FieldOwner("foo") @@ -289,6 +425,12 @@ var _ = Describe("FieldOwner", func() { t.ApplyToSubResourceUpdate(o) Expect(o.FieldManager).To(Equal("foo")) }) + It("Should apply to SubResourceApplyOptions", func() { + o := &client.SubResourceApplyOptions{ApplyOptions: client.ApplyOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourceApply(o) + Expect(o.FieldManager).To(Equal("foo")) + }) }) var _ = Describe("ForceOwnership", func() { @@ -304,6 +446,18 @@ var _ = Describe("ForceOwnership", func() { t.ApplyToSubResourcePatch(o) Expect(*o.Force).To(BeTrue()) }) + It("Should apply to ApplyOptions", func() { + o := &client.ApplyOptions{} + t := client.ForceOwnership + t.ApplyToApply(o) + Expect(*o.Force).To(BeTrue()) + }) + It("Should apply to SubResourceApplyOptions", func() { + o := &client.SubResourceApplyOptions{} + t := client.ForceOwnership + t.ApplyToSubResourceApply(o) + Expect(*o.Force).To(BeTrue()) + }) }) var _ = Describe("HasLabels", func() { diff --git a/pkg/client/patch.go b/pkg/client/patch.go index 11d6083885..9bd0953fdc 100644 --- a/pkg/client/patch.go +++ b/pkg/client/patch.go @@ -27,6 +27,8 @@ import ( var ( // Apply uses server-side apply to patch the given object. + // + // Deprecated: Use client.Client.Apply() and client.Client.SubResource("subrsource").Apply() instead. Apply Patch = applyPatch{} // Merge uses the raw object as a merge patch, without modifications. diff --git a/pkg/client/patch_test.go b/pkg/client/patch_test.go index 2910ef56bf..c9e105ae51 100644 --- a/pkg/client/patch_test.go +++ b/pkg/client/patch_test.go @@ -19,9 +19,13 @@ package client import ( "testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func BenchmarkMergeFrom(b *testing.B) { @@ -93,3 +97,26 @@ func BenchmarkMergeFrom(b *testing.B) { } }) } + +var _ = Describe("MergeFrom", func() { + It("should successfully create a patch for two large and similar in64s", func() { + var largeInt64 int64 = 9223372036854775807 + var similarLargeInt64 int64 = 9223372036854775800 + j := batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: batchv1.JobSpec{ + ActiveDeadlineSeconds: &largeInt64, + }, + } + patch := MergeFrom(j.DeepCopy()) + + j.Spec.ActiveDeadlineSeconds = &similarLargeInt64 + + data, err := patch.Data(&j) + Expect(err).NotTo(HaveOccurred()) + Expect(data).To(Equal([]byte(`{"spec":{"activeDeadlineSeconds":9223372036854775800}}`))) + }) +}) diff --git a/pkg/client/typed_client.go b/pkg/client/typed_client.go index 92afd9a9c2..66ae2e4a5c 100644 --- a/pkg/client/typed_client.go +++ b/pkg/client/typed_client.go @@ -18,8 +18,10 @@ package client import ( "context" + "fmt" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/apply" ) var _ Reader = &typedClient{} @@ -41,7 +43,7 @@ func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOpti createOpts.ApplyOptions(opts) return o.Post(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). Body(obj). VersionedParams(createOpts.AsCreateOptions(), c.paramCodec). @@ -60,9 +62,9 @@ func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOpti updateOpts.ApplyOptions(opts) return o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(obj). VersionedParams(updateOpts.AsUpdateOptions(), c.paramCodec). Do(ctx). @@ -80,9 +82,9 @@ func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOpti deleteOpts.ApplyOptions(opts) return o.Delete(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(deleteOpts.AsDeleteOptions()). Do(ctx). Error() @@ -123,15 +125,40 @@ func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts . patchOpts.ApplyOptions(opts) return o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). VersionedParams(patchOpts.AsPatchOptions(), c.paramCodec). Body(data). Do(ctx). Into(obj) } +func (c *typedClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + req, err := apply.NewRequest(o, obj) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + applyOpts := &ApplyOptions{} + applyOpts.ApplyOptions(opts) + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + VersionedParams(applyOpts.AsPatchOptions(), c.paramCodec). + Do(ctx). + // This is hacky, it is required because `Into` takes a `runtime.Object` and + // that is not implemented by the ApplyConfigurations. The generated clients + // don't have this problem because they deserialize into the api type, not the + // apply configuration: https://github.com/kubernetes/kubernetes/blob/22f5e01a37c0bc6a5f494dec14dd4e3688ee1d55/staging/src/k8s.io/client-go/gentype/type.go#L296-L317 + Into(runtimeObjectFromApplyConfiguration(obj)) +} + // Get implements client.Client. func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { r, err := c.resources.getResource(obj) @@ -179,9 +206,9 @@ func (c *typedClient) GetSubResource(ctx context.Context, obj, subResourceObj Ob getOpts.ApplyOptions(opts) return o.Get(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). VersionedParams(getOpts.AsGetOptions(), c.paramCodec). Do(ctx). @@ -202,9 +229,9 @@ func (c *typedClient) CreateSubResource(ctx context.Context, obj Object, subReso createOpts.ApplyOptions(opts) return o.Post(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). Body(subResourceObj). VersionedParams(createOpts.AsCreateOptions(), c.paramCodec). @@ -237,9 +264,9 @@ func (c *typedClient) UpdateSubResource(ctx context.Context, obj Object, subReso } return o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). Body(body). VersionedParams(updateOpts.AsUpdateOptions(), c.paramCodec). @@ -268,12 +295,45 @@ func (c *typedClient) PatchSubResource(ctx context.Context, obj Object, subResou } return o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). Body(data). VersionedParams(patchOpts.AsPatchOptions(), c.paramCodec). Do(ctx). Into(body) } + +func (c *typedClient) ApplySubResource(ctx context.Context, obj runtime.ApplyConfiguration, subResource string, opts ...SubResourceApplyOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + + applyOpts := &SubResourceApplyOptions{} + applyOpts.ApplyOpts(opts) + + body := obj + if applyOpts.SubResourceBody != nil { + body = applyOpts.SubResourceBody + } + + req, err := apply.NewRequest(o, body) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + VersionedParams(applyOpts.AsPatchOptions(), c.paramCodec). + Do(ctx). + // This is hacky, it is required because `Into` takes a `runtime.Object` and + // that is not implemented by the ApplyConfigurations. The generated clients + // don't have this problem because they deserialize into the api type, not the + // apply configuration: https://github.com/kubernetes/kubernetes/blob/22f5e01a37c0bc6a5f494dec14dd4e3688ee1d55/staging/src/k8s.io/client-go/gentype/type.go#L296-L317 + Into(runtimeObjectFromApplyConfiguration(obj)) +} diff --git a/pkg/client/unstructured_client.go b/pkg/client/unstructured_client.go index 0d96951780..d2ea6d7a32 100644 --- a/pkg/client/unstructured_client.go +++ b/pkg/client/unstructured_client.go @@ -22,6 +22,7 @@ import ( "strings" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/util/apply" ) var _ Reader = &unstructuredClient{} @@ -50,7 +51,7 @@ func (uc *unstructuredClient) Create(ctx context.Context, obj Object, opts ...Cr createOpts.ApplyOptions(opts) result := o.Post(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). Body(obj). VersionedParams(createOpts.AsCreateOptions(), uc.paramCodec). @@ -79,9 +80,9 @@ func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...Up updateOpts.ApplyOptions(opts) result := o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(obj). VersionedParams(updateOpts.AsUpdateOptions(), uc.paramCodec). Do(ctx). @@ -106,9 +107,9 @@ func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...De deleteOpts.ApplyOptions(opts) return o.Delete(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). Body(deleteOpts.AsDeleteOptions()). Do(ctx). Error() @@ -157,15 +158,41 @@ func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch patchOpts.ApplyOptions(opts) return o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). VersionedParams(patchOpts.AsPatchOptions(), uc.paramCodec). Body(data). Do(ctx). Into(obj) } +func (uc *unstructuredClient) Apply(ctx context.Context, obj runtime.ApplyConfiguration, opts ...ApplyOption) error { + unstructuredApplyConfig, ok := obj.(*unstructuredApplyConfiguration) + if !ok { + return fmt.Errorf("bug: unstructured client got an applyconfiguration that was not %T but %T", &unstructuredApplyConfiguration{}, obj) + } + o, err := uc.resources.getObjMeta(unstructuredApplyConfig.Unstructured) + if err != nil { + return err + } + + req, err := apply.NewRequest(o, obj) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + applyOpts := &ApplyOptions{} + applyOpts.ApplyOptions(opts) + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + VersionedParams(applyOpts.AsPatchOptions(), uc.paramCodec). + Do(ctx). + Into(unstructuredApplyConfig.Unstructured) +} + // Get implements client.Client. func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { u, ok := obj.(runtime.Unstructured) @@ -244,9 +271,9 @@ func (uc *unstructuredClient) GetSubResource(ctx context.Context, obj, subResour getOpts.ApplyOptions(opts) return o.Get(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). VersionedParams(getOpts.AsGetOptions(), uc.paramCodec). Do(ctx). @@ -275,9 +302,9 @@ func (uc *unstructuredClient) CreateSubResource(ctx context.Context, obj, subRes createOpts.ApplyOptions(opts) return o.Post(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). Body(subResourceObj). VersionedParams(createOpts.AsCreateOptions(), uc.paramCodec). @@ -310,9 +337,9 @@ func (uc *unstructuredClient) UpdateSubResource(ctx context.Context, obj Object, } return o.Put(). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). Body(body). VersionedParams(updateOpts.AsUpdateOptions(), uc.paramCodec). @@ -347,9 +374,9 @@ func (uc *unstructuredClient) PatchSubResource(ctx context.Context, obj Object, } result := o.Patch(patch.Type()). - NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + NamespaceIfScoped(o.namespace, o.isNamespaced()). Resource(o.resource()). - Name(o.GetName()). + Name(o.name). SubResource(subResource). Body(data). VersionedParams(patchOpts.AsPatchOptions(), uc.paramCodec). @@ -359,3 +386,35 @@ func (uc *unstructuredClient) PatchSubResource(ctx context.Context, obj Object, u.GetObjectKind().SetGroupVersionKind(gvk) return result } + +func (uc *unstructuredClient) ApplySubResource(ctx context.Context, obj runtime.ApplyConfiguration, subResource string, opts ...SubResourceApplyOption) error { + unstructuredApplyConfig, ok := obj.(*unstructuredApplyConfiguration) + if !ok { + return fmt.Errorf("bug: unstructured client got an applyconfiguration that was not %T but %T", &unstructuredApplyConfiguration{}, obj) + } + o, err := uc.resources.getObjMeta(unstructuredApplyConfig.Unstructured) + if err != nil { + return err + } + + applyOpts := &SubResourceApplyOptions{} + applyOpts.ApplyOpts(opts) + + body := obj + if applyOpts.SubResourceBody != nil { + body = applyOpts.SubResourceBody + } + req, err := apply.NewRequest(o, body) + if err != nil { + return fmt.Errorf("failed to create apply request: %w", err) + } + + return req. + NamespaceIfScoped(o.namespace, o.isNamespaced()). + Resource(o.resource()). + Name(o.name). + SubResource(subResource). + VersionedParams(applyOpts.AsPatchOptions(), uc.paramCodec). + Do(ctx). + Into(unstructuredApplyConfig.Unstructured) +} diff --git a/pkg/client/watch_test.go b/pkg/client/watch_test.go index 26d90f6550..8d5b3344d3 100644 --- a/pkg/client/watch_test.go +++ b/pkg/client/watch_test.go @@ -38,9 +38,8 @@ var _ = Describe("ClientWithWatch", func() { var count uint64 = 0 var replicaCount int32 = 2 var ns = "kube-public" - ctx := context.TODO() - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { atomic.AddUint64(&count, 1) dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("watch-deployment-name-%v", count), Namespace: ns, Labels: map[string]string{"app": fmt.Sprintf("bar-%v", count)}}, @@ -61,18 +60,18 @@ var _ = Describe("ClientWithWatch", func() { Expect(err).NotTo(HaveOccurred()) }) - AfterEach(func() { + AfterEach(func(ctx SpecContext) { deleteDeployment(ctx, dep, ns) }) Describe("NewWithWatch", func() { - It("should return a new Client", func() { + It("should return a new Client", func(ctx SpecContext) { cl, err := client.NewWithWatch(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) }) - watchSuite := func(through client.ObjectList, expectedType client.Object, checkGvk bool) { + watchSuite := func(ctx context.Context, through client.ObjectList, expectedType client.Object, checkGvk bool) { cl, err := client.NewWithWatch(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -110,23 +109,23 @@ var _ = Describe("ClientWithWatch", func() { } } - It("should receive a create event when watching the typed object", func() { - watchSuite(&appsv1.DeploymentList{}, &appsv1.Deployment{}, false) + It("should receive a create event when watching the typed object", func(ctx SpecContext) { + watchSuite(ctx, &appsv1.DeploymentList{}, &appsv1.Deployment{}, false) }) - It("should receive a create event when watching the unstructured object", func() { + It("should receive a create event when watching the unstructured object", func(ctx SpecContext) { u := &unstructured.UnstructuredList{} u.SetGroupVersionKind(schema.GroupVersionKind{ Group: "apps", Kind: "Deployment", Version: "v1", }) - watchSuite(u, &unstructured.Unstructured{}, true) + watchSuite(ctx, u, &unstructured.Unstructured{}, true) }) - It("should receive a create event when watching the metadata object", func() { + It("should receive a create event when watching the metadata object", func(ctx SpecContext) { m := &metav1.PartialObjectMetadataList{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}} - watchSuite(m, &metav1.PartialObjectMetadata{}, false) + watchSuite(ctx, m, &metav1.PartialObjectMetadata{}, false) }) }) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 7d00c3c4b0..0603f4cde5 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -20,7 +20,6 @@ import ( "context" "errors" "net/http" - "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/meta" @@ -28,12 +27,11 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" - "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" ) @@ -66,8 +64,8 @@ type Cluster interface { // GetRESTMapper returns a RESTMapper GetRESTMapper() meta.RESTMapper - // GetAPIReader returns a reader that will be configured to use the API server. - // This should be used sparingly and only when the client does not fit your + // GetAPIReader returns a reader that will be configured to use the API server directly. + // This should be used sparingly and only when the cached client does not fit your // use case. GetAPIReader() client.Reader @@ -89,24 +87,6 @@ type Options struct { // If none is set, it defaults to log.Log global logger. Logger logr.Logger - // SyncPeriod determines the minimum frequency at which watched resources are - // reconciled. A lower period will correct entropy more quickly, but reduce - // responsiveness to change if there are many watched resources. Change this - // value only if you know what you are doing. Defaults to 10 hours if unset. - // there will a 10 percent jitter between the SyncPeriod of all controllers - // so that all controllers will not send list requests simultaneously. - SyncPeriod *time.Duration - - // Namespace if specified restricts the manager's cache to watch objects in - // the desired namespace Defaults to all namespaces - // - // Note: If a namespace is specified, controllers can still Watch for a - // cluster-scoped resource (e.g Node). For namespaced resources the cache - // will only hold objects from the desired namespace. - // - // Deprecated: Use Cache.Namespaces instead. - Namespace string - // HTTPClient is the http client that will be used to create the default // Cache and Client. If not set the rest.HTTPClientFor function will be used // to create the http client. @@ -141,18 +121,6 @@ type Options struct { // Only use a custom NewClient if you know what you are doing. NewClient client.NewClientFunc - // ClientDisableCacheFor tells the client that, if any cache is used, to bypass it - // for the given objects. - // - // Deprecated: Use Client.Cache.DisableFor instead. - ClientDisableCacheFor []client.Object - - // DryRunClient specifies whether the client should be configured to enforce - // dryRun mode. - // - // Deprecated: Use Client.DryRun instead. - DryRunClient bool - // EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API // Use this to customize the event correlator and spam filter // @@ -179,6 +147,13 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { return nil, errors.New("must specify Config") } + originalConfig := config + + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + options := Options{} for _, opt := range opts { opt(&options) @@ -208,12 +183,6 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { if cacheOpts.HTTPClient == nil { cacheOpts.HTTPClient = options.HTTPClient } - if cacheOpts.SyncPeriod == nil { - cacheOpts.SyncPeriod = options.SyncPeriod - } - if len(cacheOpts.Namespaces) == 0 && options.Namespace != "" { - cacheOpts.Namespaces = []string{options.Namespace} - } } cache, err := options.NewCache(config, cacheOpts) if err != nil { @@ -240,16 +209,6 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { if clientOpts.Cache.Reader == nil { clientOpts.Cache.Reader = cache } - - // For backward compatibility, the ClientDisableCacheFor option should - // be appended to the DisableFor option in the client. - clientOpts.Cache.DisableFor = append(clientOpts.Cache.DisableFor, options.ClientDisableCacheFor...) - - if clientOpts.DryRun == nil && options.DryRunClient { - // For backward compatibility, the DryRunClient (if set) option should override - // the DryRun option in the client (if unset). - clientOpts.DryRun = pointer.Bool(true) - } } clientWriter, err := options.NewClient(config, clientOpts) if err != nil { @@ -275,7 +234,7 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { } return &cluster{ - config: config, + config: originalConfig, httpClient: options.HTTPClient, scheme: options.Scheme, cache: cache, diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index dc52b2d9b3..c08a742403 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -100,22 +100,22 @@ var _ = Describe("cluster.Cluster", func() { }) Describe("Start", func() { - It("should stop when context is cancelled", func() { + It("should stop when context is cancelled", func(specCtx SpecContext) { c, err := New(cfg) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(c.Start(ctx)).NotTo(HaveOccurred()) }) }) - It("should not leak goroutines when stopped", func() { + It("should not leak goroutines when stopped", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() c, err := New(cfg) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(c.Start(ctx)).NotTo(HaveOccurred()) diff --git a/pkg/config/config.go b/pkg/config/config.go deleted file mode 100644 index 9c7b875a86..0000000000 --- a/pkg/config/config.go +++ /dev/null @@ -1,112 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "fmt" - "os" - "sync" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" -) - -// ControllerManagerConfiguration defines the functions necessary to parse a config file -// and to configure the Options struct for the ctrl.Manager. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -type ControllerManagerConfiguration interface { - runtime.Object - - // Complete returns the versioned configuration - Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) -} - -// DeferredFileLoader is used to configure the decoder for loading controller -// runtime component config types. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -type DeferredFileLoader struct { - ControllerManagerConfiguration - path string - scheme *runtime.Scheme - once sync.Once - err error -} - -// File will set up the deferred file loader for the configuration -// this will also configure the defaults for the loader if nothing is -// -// Defaults: -// * Path: "./config.yaml" -// * Kind: GenericControllerManagerConfiguration -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -func File() *DeferredFileLoader { - scheme := runtime.NewScheme() - utilruntime.Must(v1alpha1.AddToScheme(scheme)) - return &DeferredFileLoader{ - path: "./config.yaml", - ControllerManagerConfiguration: &v1alpha1.ControllerManagerConfiguration{}, - scheme: scheme, - } -} - -// Complete will use sync.Once to set the scheme. -func (d *DeferredFileLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { - d.once.Do(d.loadFile) - if d.err != nil { - return v1alpha1.ControllerManagerConfigurationSpec{}, d.err - } - return d.ControllerManagerConfiguration.Complete() -} - -// AtPath will set the path to load the file for the decoder. -func (d *DeferredFileLoader) AtPath(path string) *DeferredFileLoader { - d.path = path - return d -} - -// OfKind will set the type to be used for decoding the file into. -func (d *DeferredFileLoader) OfKind(obj ControllerManagerConfiguration) *DeferredFileLoader { - d.ControllerManagerConfiguration = obj - return d -} - -// loadFile is used from the mutex.Once to load the file. -func (d *DeferredFileLoader) loadFile() { - if d.scheme == nil { - d.err = fmt.Errorf("scheme not supplied to controller configuration loader") - return - } - - content, err := os.ReadFile(d.path) - if err != nil { - d.err = fmt.Errorf("could not read file at %s", d.path) - return - } - - codecs := serializer.NewCodecFactory(d.scheme) - - // Regardless of if the bytes are of any external version, - // it will be read successfully and converted into the internal version - if err = runtime.DecodeInto(codecs.UniversalDecoder(), content, d.ControllerManagerConfiguration); err != nil { - d.err = fmt.Errorf("could not decode file into runtime.Object") - } -} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go deleted file mode 100644 index a38de41076..0000000000 --- a/pkg/config/config_test.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" -) - -var _ = Describe("config", func() { - Describe("File", func() { - - It("should error loading from non existent file", func() { - loader := config.File() - _, err := loader.Complete() - Expect(err).To(HaveOccurred()) - }) - - It("should load a config from file", func() { - conf := v1alpha1.ControllerManagerConfiguration{} - loader := config.File().AtPath("./testdata/config.yaml").OfKind(&conf) - Expect(conf.CacheNamespace).To(Equal("")) - - _, err := loader.Complete() - Expect(err).ToNot(HaveOccurred()) - - Expect(*conf.LeaderElection.LeaderElect).To(BeTrue()) - Expect(conf.CacheNamespace).To(Equal("default")) - Expect(conf.Metrics.BindAddress).To(Equal(":8081")) - }) - }) -}) diff --git a/pkg/config/controller.go b/pkg/config/controller.go index b37dffaeea..5eea2965f6 100644 --- a/pkg/config/controller.go +++ b/pkg/config/controller.go @@ -16,10 +16,22 @@ limitations under the License. package config -import "time" +import ( + "time" -// Controller contains configuration options for a controller. + "github.com/go-logr/logr" +) + +// Controller contains configuration options for controllers. It only includes options +// that makes sense for a set of controllers and is used for defaulting the options +// of multiple controllers. type Controller struct { + // SkipNameValidation allows skipping the name validation that ensures that every controller name is unique. + // Unique controller names are important to get unique metrics and logs for a controller. + // Can be overwritten for a controller via the SkipNameValidation setting on the controller. + // Defaults to false if SkipNameValidation setting on controller and Manager are unset. + SkipNameValidation *bool + // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation // allowed for that controller. // @@ -40,10 +52,41 @@ type Controller struct { CacheSyncTimeout time.Duration // RecoverPanic indicates whether the panic caused by reconcile should be recovered. - // Defaults to the Controller.RecoverPanic setting from the Manager if unset. + // Can be overwritten for a controller via the RecoverPanic setting on the controller. + // Defaults to true if RecoverPanic setting on controller and Manager are unset. RecoverPanic *bool // NeedLeaderElection indicates whether the controller needs to use leader election. // Defaults to true, which means the controller will use leader election. NeedLeaderElection *bool + + // EnableWarmup specifies whether the controller should start its sources when the manager is not + // the leader. This is useful for cases where sources take a long time to start, as it allows + // for the controller to warm up its caches even before it is elected as the leader. This + // improves leadership failover time, as the caches will be prepopulated before the controller + // transitions to be leader. + // + // Setting EnableWarmup to true and NeedLeaderElection to true means the controller will start its + // sources without waiting to become leader. + // Setting EnableWarmup to true and NeedLeaderElection to false is a no-op as controllers without + // leader election do not wait on leader election to start their sources. + // Defaults to false. + // + // Note: This feature is currently in beta and subject to change. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/3220. + EnableWarmup *bool + + // UsePriorityQueue configures the controllers queue to use the controller-runtime provided + // priority queue. + // + // Note: This flag is enabled by default. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/2374. + UsePriorityQueue *bool + + // Logger is the logger controllers should use. + Logger logr.Logger + + // ReconciliationTimeout is used as the timeout passed to the context of each Reconcile call. + // By default, there is no timeout. + ReconciliationTimeout time.Duration } diff --git a/pkg/config/example_test.go b/pkg/config/example_test.go deleted file mode 100644 index 3d80d68eff..0000000000 --- a/pkg/config/example_test.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config_test - -import ( - "fmt" - "os" - - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/config" - - "sigs.k8s.io/controller-runtime/examples/configfile/custom/v1alpha1" -) - -var scheme = runtime.NewScheme() - -func init() { - _ = v1alpha1.AddToScheme(scheme) -} - -// This example will load a file using Complete with only -// defaults set. -func ExampleFile() { - // This will load a config file from ./config.yaml - loader := config.File() - if _, err := loader.Complete(); err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} - -// This example will load the file from a custom path. -func ExampleFile_atPath() { - loader := config.File().AtPath("/var/run/controller-runtime/config.yaml") - if _, err := loader.Complete(); err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} diff --git a/pkg/config/testdata/config.yaml b/pkg/config/testdata/config.yaml deleted file mode 100644 index d88da3a65b..0000000000 --- a/pkg/config/testdata/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 -kind: ControllerManagerConfiguration -cacheNamespace: default -metrics: - bindAddress: :8081 -leaderElection: - leaderElect: true diff --git a/pkg/config/v1alpha1/doc.go b/pkg/config/v1alpha1/doc.go deleted file mode 100644 index 8fdf14d39a..0000000000 --- a/pkg/config/v1alpha1/doc.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1alpha1 provides the ControllerManagerConfiguration used for -// configuring ctrl.Manager -// +kubebuilder:object:generate=true -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -package v1alpha1 diff --git a/pkg/config/v1alpha1/register.go b/pkg/config/v1alpha1/register.go deleted file mode 100644 index ca854bcf30..0000000000 --- a/pkg/config/v1alpha1/register.go +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects. - // - // Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. - GroupVersion = schema.GroupVersion{Group: "controller-runtime.sigs.k8s.io", Version: "v1alpha1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme. - // - // Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - // - // Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. - AddToScheme = SchemeBuilder.AddToScheme -) - -func init() { - SchemeBuilder.Register(&ControllerManagerConfiguration{}) -} diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go deleted file mode 100644 index 52c8ab300f..0000000000 --- a/pkg/config/v1alpha1/types.go +++ /dev/null @@ -1,179 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - configv1alpha1 "k8s.io/component-base/config/v1alpha1" -) - -// ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -type ControllerManagerConfigurationSpec struct { - // SyncPeriod determines the minimum frequency at which watched resources are - // reconciled. A lower period will correct entropy more quickly, but reduce - // responsiveness to change if there are many watched resources. Change this - // value only if you know what you are doing. Defaults to 10 hours if unset. - // there will a 10 percent jitter between the SyncPeriod of all controllers - // so that all controllers will not send list requests simultaneously. - // +optional - SyncPeriod *metav1.Duration `json:"syncPeriod,omitempty"` - - // LeaderElection is the LeaderElection config to be used when configuring - // the manager.Manager leader election - // +optional - LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"` - - // CacheNamespace if specified restricts the manager's cache to watch objects in - // the desired namespace Defaults to all namespaces - // - // Note: If a namespace is specified, controllers can still Watch for a - // cluster-scoped resource (e.g Node). For namespaced resources the cache - // will only hold objects from the desired namespace. - // +optional - CacheNamespace string `json:"cacheNamespace,omitempty"` - - // GracefulShutdownTimeout is the duration given to runnable to stop before the manager actually returns on stop. - // To disable graceful shutdown, set to time.Duration(0) - // To use graceful shutdown without timeout, set to a negative duration, e.G. time.Duration(-1) - // The graceful shutdown is skipped for safety reasons in case the leader election lease is lost. - GracefulShutdownTimeout *metav1.Duration `json:"gracefulShutDown,omitempty"` - - // Controller contains global configuration options for controllers - // registered within this manager. - // +optional - Controller *ControllerConfigurationSpec `json:"controller,omitempty"` - - // Metrics contains the controller metrics configuration - // +optional - Metrics ControllerMetrics `json:"metrics,omitempty"` - - // Health contains the controller health configuration - // +optional - Health ControllerHealth `json:"health,omitempty"` - - // Webhook contains the controllers webhook configuration - // +optional - Webhook ControllerWebhook `json:"webhook,omitempty"` -} - -// ControllerConfigurationSpec defines the global configuration for -// controllers registered with the manager. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -// -// Deprecated: Controller global configuration can now be set at the manager level, -// using the manager.Options.Controller field. -type ControllerConfigurationSpec struct { - // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation - // allowed for that controller. - // - // When a controller is registered within this manager using the builder utilities, - // users have to specify the type the controller reconciles in the For(...) call. - // If the object's kind passed matches one of the keys in this map, the concurrency - // for that controller is set to the number specified. - // - // The key is expected to be consistent in form with GroupKind.String(), - // e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`. - // - // +optional - GroupKindConcurrency map[string]int `json:"groupKindConcurrency,omitempty"` - - // CacheSyncTimeout refers to the time limit set to wait for syncing caches. - // Defaults to 2 minutes if not set. - // +optional - CacheSyncTimeout *time.Duration `json:"cacheSyncTimeout,omitempty"` - - // RecoverPanic indicates if panics should be recovered. - // +optional - RecoverPanic *bool `json:"recoverPanic,omitempty"` -} - -// ControllerMetrics defines the metrics configs. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -type ControllerMetrics struct { - // BindAddress is the TCP address that the controller should bind to - // for serving prometheus metrics. - // It can be set to "0" to disable the metrics serving. - // +optional - BindAddress string `json:"bindAddress,omitempty"` -} - -// ControllerHealth defines the health configs. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -type ControllerHealth struct { - // HealthProbeBindAddress is the TCP address that the controller should bind to - // for serving health probes - // It can be set to "0" or "" to disable serving the health probe. - // +optional - HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"` - - // ReadinessEndpointName, defaults to "readyz" - // +optional - ReadinessEndpointName string `json:"readinessEndpointName,omitempty"` - - // LivenessEndpointName, defaults to "healthz" - // +optional - LivenessEndpointName string `json:"livenessEndpointName,omitempty"` -} - -// ControllerWebhook defines the webhook server for the controller. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -type ControllerWebhook struct { - // Port is the port that the webhook server serves at. - // It is used to set webhook.Server.Port. - // +optional - Port *int `json:"port,omitempty"` - - // Host is the hostname that the webhook server binds to. - // It is used to set webhook.Server.Host. - // +optional - Host string `json:"host,omitempty"` - - // CertDir is the directory that contains the server key and certificate. - // if not set, webhook server would look up the server key and certificate in - // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate - // must be named tls.key and tls.crt, respectively. - // +optional - CertDir string `json:"certDir,omitempty"` -} - -// +kubebuilder:object:root=true - -// ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -type ControllerManagerConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // ControllerManagerConfiguration returns the contfigurations for controllers - ControllerManagerConfigurationSpec `json:",inline"` -} - -// Complete returns the configuration for controller-runtime. -// -// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. -func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) { - return *c, nil -} diff --git a/pkg/config/v1alpha1/zz_generated.deepcopy.go b/pkg/config/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index cdc7c334be..0000000000 --- a/pkg/config/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,158 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - configv1alpha1 "k8s.io/component-base/config/v1alpha1" - timex "time" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerConfigurationSpec) DeepCopyInto(out *ControllerConfigurationSpec) { - *out = *in - if in.GroupKindConcurrency != nil { - in, out := &in.GroupKindConcurrency, &out.GroupKindConcurrency - *out = make(map[string]int, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.CacheSyncTimeout != nil { - in, out := &in.CacheSyncTimeout, &out.CacheSyncTimeout - *out = new(timex.Duration) - **out = **in - } - if in.RecoverPanic != nil { - in, out := &in.RecoverPanic, &out.RecoverPanic - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerConfigurationSpec. -func (in *ControllerConfigurationSpec) DeepCopy() *ControllerConfigurationSpec { - if in == nil { - return nil - } - out := new(ControllerConfigurationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth. -func (in *ControllerHealth) DeepCopy() *ControllerHealth { - if in == nil { - return nil - } - out := new(ControllerHealth) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerManagerConfiguration) DeepCopyInto(out *ControllerManagerConfiguration) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfiguration. -func (in *ControllerManagerConfiguration) DeepCopy() *ControllerManagerConfiguration { - if in == nil { - return nil - } - out := new(ControllerManagerConfiguration) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ControllerManagerConfiguration) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerManagerConfigurationSpec) DeepCopyInto(out *ControllerManagerConfigurationSpec) { - *out = *in - if in.SyncPeriod != nil { - in, out := &in.SyncPeriod, &out.SyncPeriod - *out = new(v1.Duration) - **out = **in - } - if in.LeaderElection != nil { - in, out := &in.LeaderElection, &out.LeaderElection - *out = new(configv1alpha1.LeaderElectionConfiguration) - (*in).DeepCopyInto(*out) - } - if in.GracefulShutdownTimeout != nil { - in, out := &in.GracefulShutdownTimeout, &out.GracefulShutdownTimeout - *out = new(v1.Duration) - **out = **in - } - if in.Controller != nil { - in, out := &in.Controller, &out.Controller - *out = new(ControllerConfigurationSpec) - (*in).DeepCopyInto(*out) - } - out.Metrics = in.Metrics - out.Health = in.Health - in.Webhook.DeepCopyInto(&out.Webhook) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfigurationSpec. -func (in *ControllerManagerConfigurationSpec) DeepCopy() *ControllerManagerConfigurationSpec { - if in == nil { - return nil - } - out := new(ControllerManagerConfigurationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics. -func (in *ControllerMetrics) DeepCopy() *ControllerMetrics { - if in == nil { - return nil - } - out := new(ControllerMetrics) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) { - *out = *in - if in.Port != nil { - in, out := &in.Port, &out.Port - *out = new(int) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook. -func (in *ControllerWebhook) DeepCopy() *ControllerWebhook { - if in == nil { - return nil - } - out := new(ControllerWebhook) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e48db41f94..853788d52f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -24,18 +24,27 @@ import ( "github.com/go-logr/logr" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" + "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/internal/controller" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/ratelimiter" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) // Options are the arguments for creating a new Controller. -type Options struct { +type Options = TypedOptions[reconcile.Request] + +// TypedOptions are the arguments for creating a new Controller. +type TypedOptions[request comparable] struct { + // SkipNameValidation allows skipping the name validation that ensures that every controller name is unique. + // Unique controller names are important to get unique metrics and logs for a controller. + // Defaults to the Controller.SkipNameValidation setting from the Manager if unset. + // Defaults to false if Controller.SkipNameValidation setting from the Manager is also unset. + SkipNameValidation *bool + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. MaxConcurrentReconciles int @@ -45,6 +54,7 @@ type Options struct { // RecoverPanic indicates whether the panic caused by reconcile should be recovered. // Defaults to the Controller.RecoverPanic setting from the Manager if unset. + // Defaults to true if Controller.RecoverPanic setting from the Manager is also unset. RecoverPanic *bool // NeedLeaderElection indicates whether the controller needs to use leader election. @@ -52,33 +62,112 @@ type Options struct { NeedLeaderElection *bool // Reconciler reconciles an object - Reconciler reconcile.Reconciler + Reconciler reconcile.TypedReconciler[request] // RateLimiter is used to limit how frequently requests may be queued. // Defaults to MaxOfRateLimiter which has both overall and per-item rate limiting. // The overall is a token bucket and the per-item is exponential. - RateLimiter ratelimiter.RateLimiter + RateLimiter workqueue.TypedRateLimiter[request] + + // NewQueue constructs the queue for this controller once the controller is ready to start. + // With NewQueue a custom queue implementation can be used, e.g. a priority queue to prioritize with which + // priority/order objects are reconciled (e.g. to reconcile objects with changes first). + // This is a func because the standard Kubernetes work queues start themselves immediately, which + // leads to goroutine leaks if something calls controller.New repeatedly. + // The NewQueue func gets the controller name and the RateLimiter option (defaulted if necessary) passed in. + // NewQueue defaults to NewRateLimitingQueueWithConfig. + // + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewQueue if you know what you are doing. + NewQueue func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] + + // Logger will be used to build a default LogConstructor if unset. + Logger logr.Logger // LogConstructor is used to construct a logger used for this controller and passed // to each reconciliation via the context field. - LogConstructor func(request *reconcile.Request) logr.Logger + LogConstructor func(request *request) logr.Logger + + // UsePriorityQueue configures the controllers queue to use the controller-runtime provided + // priority queue. + // + // Note: This flag is enabled by default. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/2374. + UsePriorityQueue *bool + + // EnableWarmup specifies whether the controller should start its sources when the manager is not + // the leader. This is useful for cases where sources take a long time to start, as it allows + // for the controller to warm up its caches even before it is elected as the leader. This + // improves leadership failover time, as the caches will be prepopulated before the controller + // transitions to be leader. + // + // Setting EnableWarmup to true and NeedLeaderElection to true means the controller will start its + // sources without waiting to become leader. + // Setting EnableWarmup to true and NeedLeaderElection to false is a no-op as controllers without + // leader election do not wait on leader election to start their sources. + // Defaults to false. + // + // Note: This feature is currently in beta and subject to change. + // For more details, see: https://github.com/kubernetes-sigs/controller-runtime/issues/3220. + EnableWarmup *bool + + // ReconciliationTimeout is used as the timeout passed to the context of each Reconcile call. + // By default, there is no timeout. + ReconciliationTimeout time.Duration } -// Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests -// from source.Sources. Work is performed through the reconcile.Reconciler for each enqueued item. +// DefaultFromConfig defaults the config from a config.Controller +func (options *TypedOptions[request]) DefaultFromConfig(config config.Controller) { + if options.Logger.GetSink() == nil { + options.Logger = config.Logger + } + + if options.SkipNameValidation == nil { + options.SkipNameValidation = config.SkipNameValidation + } + + if options.MaxConcurrentReconciles <= 0 && config.MaxConcurrentReconciles > 0 { + options.MaxConcurrentReconciles = config.MaxConcurrentReconciles + } + + if options.CacheSyncTimeout == 0 && config.CacheSyncTimeout > 0 { + options.CacheSyncTimeout = config.CacheSyncTimeout + } + + if options.UsePriorityQueue == nil { + options.UsePriorityQueue = config.UsePriorityQueue + } + + if options.RecoverPanic == nil { + options.RecoverPanic = config.RecoverPanic + } + + if options.NeedLeaderElection == nil { + options.NeedLeaderElection = config.NeedLeaderElection + } + + if options.EnableWarmup == nil { + options.EnableWarmup = config.EnableWarmup + } + + if options.ReconciliationTimeout == 0 { + options.ReconciliationTimeout = config.ReconciliationTimeout + } +} + +// Controller implements an API. A Controller manages a work queue fed reconcile.Requests +// from source.Sources. Work is performed through the reconcile.Reconciler for each enqueued item. // Work typically is reads and writes Kubernetes objects to make the system state match the state specified // in the object Spec. -type Controller interface { +type Controller = TypedController[reconcile.Request] + +// TypedController implements an API. +type TypedController[request comparable] interface { // Reconciler is called to reconcile an object by Namespace/Name - reconcile.Reconciler + reconcile.TypedReconciler[request] - // Watch takes events provided by a Source and uses the EventHandler to - // enqueue reconcile.Requests in response to the events. - // - // Watch may be provided one or more Predicates to filter events before - // they are given to the EventHandler. Events will be passed to the - // EventHandler if all provided Predicates evaluate to true. - Watch(src source.Source, eventhandler handler.EventHandler, predicates ...predicate.Predicate) error + // Watch watches the provided Source. + Watch(src source.TypedSource[request]) error // Start starts the controller. Start blocks until the context is closed or a // controller has an error starting. @@ -90,8 +179,18 @@ type Controller interface { // New returns a new Controller registered with the Manager. The Manager will ensure that shared Caches have // been synced before the Controller is Started. +// +// The name must be unique as it is used to identify the controller in metrics and logs. func New(name string, mgr manager.Manager, options Options) (Controller, error) { - c, err := NewUnmanaged(name, mgr, options) + return NewTyped(name, mgr, options) +} + +// NewTyped returns a new typed controller registered with the Manager, +// +// The name must be unique as it is used to identify the controller in metrics and logs. +func NewTyped[request comparable](name string, mgr manager.Manager, options TypedOptions[request]) (TypedController[request], error) { + options.DefaultFromConfig(mgr.GetControllerOptions()) + c, err := NewTypedUnmanaged(name, options) if err != nil { return nil, err } @@ -102,7 +201,16 @@ func New(name string, mgr manager.Manager, options Options) (Controller, error) // NewUnmanaged returns a new controller without adding it to the manager. The // caller is responsible for starting the returned controller. -func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller, error) { +// +// The name must be unique as it is used to identify the controller in metrics and logs. +func NewUnmanaged(name string, options Options) (Controller, error) { + return NewTypedUnmanaged(name, options) +} + +// NewTypedUnmanaged returns a new typed controller without adding it to the manager. +// +// The name must be unique as it is used to identify the controller in metrics and logs. +func NewTypedUnmanaged[request comparable](name string, options TypedOptions[request]) (TypedController[request], error) { if options.Reconciler == nil { return nil, fmt.Errorf("must specify Reconciler") } @@ -111,13 +219,19 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller return nil, fmt.Errorf("must specify Name for Controller") } + if options.SkipNameValidation == nil || !*options.SkipNameValidation { + if err := checkName(name); err != nil { + return nil, err + } + } + if options.LogConstructor == nil { - log := mgr.GetLogger().WithValues( + log := options.Logger.WithValues( "controller", name, ) - options.LogConstructor = func(req *reconcile.Request) logr.Logger { + options.LogConstructor = func(in *request) logr.Logger { log := log - if req != nil { + if req, ok := any(in).(*reconcile.Request); ok && req != nil { log = log.WithValues( "object", klog.KRef(req.Namespace, req.Name), "namespace", req.Namespace, "name", req.Name, @@ -128,48 +242,49 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller } if options.MaxConcurrentReconciles <= 0 { - if mgr.GetControllerOptions().MaxConcurrentReconciles > 0 { - options.MaxConcurrentReconciles = mgr.GetControllerOptions().MaxConcurrentReconciles - } else { - options.MaxConcurrentReconciles = 1 - } + options.MaxConcurrentReconciles = 1 } if options.CacheSyncTimeout == 0 { - if mgr.GetControllerOptions().CacheSyncTimeout != 0 { - options.CacheSyncTimeout = mgr.GetControllerOptions().CacheSyncTimeout - } else { - options.CacheSyncTimeout = 2 * time.Minute - } + options.CacheSyncTimeout = 2 * time.Minute } if options.RateLimiter == nil { - options.RateLimiter = workqueue.DefaultControllerRateLimiter() - } - - if options.RecoverPanic == nil { - options.RecoverPanic = mgr.GetControllerOptions().RecoverPanic + if ptr.Deref(options.UsePriorityQueue, true) { + options.RateLimiter = workqueue.NewTypedItemExponentialFailureRateLimiter[request](5*time.Millisecond, 1000*time.Second) + } else { + options.RateLimiter = workqueue.DefaultTypedControllerRateLimiter[request]() + } } - if options.NeedLeaderElection == nil { - options.NeedLeaderElection = mgr.GetControllerOptions().NeedLeaderElection + if options.NewQueue == nil { + options.NewQueue = func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] { + if ptr.Deref(options.UsePriorityQueue, true) { + return priorityqueue.New(controllerName, func(o *priorityqueue.Opts[request]) { + o.Log = options.Logger.WithValues("controller", controllerName) + o.RateLimiter = rateLimiter + }) + } + return workqueue.NewTypedRateLimitingQueueWithConfig(rateLimiter, workqueue.TypedRateLimitingQueueConfig[request]{ + Name: controllerName, + }) + } } // Create controller with dependencies set - return &controller.Controller{ - Do: options.Reconciler, - MakeQueue: func() workqueue.RateLimitingInterface { - return workqueue.NewRateLimitingQueueWithConfig(options.RateLimiter, workqueue.RateLimitingQueueConfig{ - Name: name, - }) - }, + return controller.New[request](controller.Options[request]{ + Do: options.Reconciler, + RateLimiter: options.RateLimiter, + NewQueue: options.NewQueue, MaxConcurrentReconciles: options.MaxConcurrentReconciles, CacheSyncTimeout: options.CacheSyncTimeout, Name: name, LogConstructor: options.LogConstructor, RecoverPanic: options.RecoverPanic, LeaderElected: options.NeedLeaderElection, - }, nil + EnableWarmup: options.EnableWarmup, + ReconciliationTimeout: options.ReconciliationTimeout, + }), nil } // ReconcileIDFromContext gets the reconcileID from the current context. diff --git a/pkg/controller/controller_integration_test.go b/pkg/controller/controller_integration_test.go index 48facf1e94..e09813eee2 100644 --- a/pkg/controller/controller_integration_test.go +++ b/pkg/controller/controller_integration_test.go @@ -18,16 +18,21 @@ package controller_test import ( "context" + "fmt" + "strconv" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" @@ -38,7 +43,6 @@ import ( var _ = Describe("controller", func() { var reconciled chan reconcile.Request - ctx := context.Background() BeforeEach(func() { reconciled = make(chan reconcile.Request) @@ -48,29 +52,48 @@ var _ = Describe("controller", func() { Describe("controller", func() { // TODO(directxman12): write a whole suite of controller-client interaction tests - It("should reconcile", func() { + // The watches in this test are setup with a namespace predicate to avoid each table entry + // from interfering with the others. We cannot add a delete call for the pods created in the + // test, as it causes flakes with the api-server termination timing out. + // See https://github.com/kubernetes-sigs/controller-runtime/issues/1571 for a description + // of the issue, and a discussion here: https://github.com/kubernetes-sigs/controller-runtime/pull/3192#discussion_r2186967799 + DescribeTable("should reconcile", func(ctx SpecContext, enableWarmup bool) { By("Creating the Manager") cm, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) By("Creating the Controller") - instance, err := controller.New("foo-controller", cm, controller.Options{ - Reconciler: reconcile.Func( - func(_ context.Context, request reconcile.Request) (reconcile.Result, error) { - reconciled <- request - return reconcile.Result{}, nil - }), - }) + instance, err := controller.New( + fmt.Sprintf("foo-controller-%t", enableWarmup), + cm, + controller.Options{ + Reconciler: reconcile.Func( + func(_ context.Context, request reconcile.Request) (reconcile.Result, error) { + reconciled <- request + return reconcile.Result{}, nil + }), + EnableWarmup: ptr.To(enableWarmup), + }, + ) Expect(err).NotTo(HaveOccurred()) + testNamespace := strconv.FormatBool(enableWarmup) + By("Watching Resources") err = instance.Watch( - source.Kind(cm.GetCache(), &appsv1.ReplicaSet{}), - handler.EnqueueRequestForOwner(cm.GetScheme(), cm.GetRESTMapper(), &appsv1.Deployment{}), + source.Kind(cm.GetCache(), &appsv1.ReplicaSet{}, + handler.TypedEnqueueRequestForOwner[*appsv1.ReplicaSet](cm.GetScheme(), cm.GetRESTMapper(), &appsv1.Deployment{}), + makeNamespacePredicate[*appsv1.ReplicaSet](testNamespace), + ), ) Expect(err).NotTo(HaveOccurred()) - err = instance.Watch(source.Kind(cm.GetCache(), &appsv1.Deployment{}), &handler.EnqueueRequestForObject{}) + err = instance.Watch( + source.Kind(cm.GetCache(), &appsv1.Deployment{}, + &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{}, + makeNamespacePredicate[*appsv1.Deployment](testNamespace), + ), + ) Expect(err).NotTo(HaveOccurred()) err = cm.GetClient().Get(ctx, types.NamespacedName{Name: "foo"}, &corev1.Namespace{}) @@ -79,8 +102,6 @@ var _ = Describe("controller", func() { Expect(err).To(Equal(&cache.ErrCacheNotStarted{})) By("Starting the Manager") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(cm.Start(ctx)).NotTo(HaveOccurred()) @@ -109,19 +130,25 @@ var _ = Describe("controller", func() { }, } expectedReconcileRequest := reconcile.Request{NamespacedName: types.NamespacedName{ - Namespace: "default", + Namespace: testNamespace, Name: "deployment-name", }} + By("Creating the test namespace") + _, err = clientset.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: testNamespace}, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Invoking Reconciling for Create") - deployment, err = clientset.AppsV1().Deployments("default").Create(ctx, deployment, metav1.CreateOptions{}) + deployment, err = clientset.AppsV1().Deployments(testNamespace).Create(ctx, deployment, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for Update") newDeployment := deployment.DeepCopy() newDeployment.Labels = map[string]string{"foo": "bar"} - _, err = clientset.AppsV1().Deployments("default").Update(ctx, newDeployment, metav1.UpdateOptions{}) + _, err = clientset.AppsV1().Deployments(testNamespace).Update(ctx, newDeployment, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) @@ -144,36 +171,81 @@ var _ = Describe("controller", func() { Template: deployment.Spec.Template, }, } - replicaset, err = clientset.AppsV1().ReplicaSets("default").Create(ctx, replicaset, metav1.CreateOptions{}) + replicaset, err = clientset.AppsV1().ReplicaSets(testNamespace).Create(ctx, replicaset, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for an OwnedObject when it is updated") newReplicaset := replicaset.DeepCopy() newReplicaset.Labels = map[string]string{"foo": "bar"} - _, err = clientset.AppsV1().ReplicaSets("default").Update(ctx, newReplicaset, metav1.UpdateOptions{}) + _, err = clientset.AppsV1().ReplicaSets(testNamespace).Update(ctx, newReplicaset, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for an OwnedObject when it is deleted") - err = clientset.AppsV1().ReplicaSets("default").Delete(ctx, replicaset.Name, metav1.DeleteOptions{}) + err = clientset.AppsV1().ReplicaSets(testNamespace).Delete(ctx, replicaset.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Invoking Reconciling for Delete") - err = clientset.AppsV1().Deployments("default"). + err = clientset.AppsV1().Deployments(testNamespace). Delete(ctx, "deployment-name", metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(<-reconciled).To(Equal(expectedReconcileRequest)) By("Listing a type with a slice of pointers as items field") err = cm.GetClient(). - List(context.Background(), &controllertest.UnconventionalListTypeList{}) + List(ctx, &controllertest.UnconventionalListTypeList{}) + Expect(err).NotTo(HaveOccurred()) + + By("Invoking Reconciling for a pod when it is created when adding watcher dynamically") + // Add new watcher dynamically + err = instance.Watch( + source.Kind(cm.GetCache(), &corev1.Pod{}, + &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}, + makeNamespacePredicate[*corev1.Pod](testNamespace), + ), + ) + Expect(err).NotTo(HaveOccurred()) + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "pod-name"}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:latest", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 80, + }, + }, + }, + }, + }, + } + expectedReconcileRequest = reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: testNamespace, + Name: "pod-name", + }} + _, err = clientset.CoreV1().Pods(testNamespace).Create(ctx, pod, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - }) + Expect(<-reconciled).To(Equal(expectedReconcileRequest)) + }, + Entry("with controller warmup enabled", true), + Entry("with controller warmup not enabled", false), + ) }) }) +// makeNamespacePredicate returns a predicate that filters out all objects not in the passed in +// namespace. +func makeNamespacePredicate[object client.Object](namespace string) predicate.TypedPredicate[object] { + return predicate.NewTypedPredicateFuncs[object](func(obj object) bool { + return obj.GetNamespace() == namespace + }) +} + func truePtr() *bool { t := true return &t diff --git a/pkg/controller/controller_suite_test.go b/pkg/controller/controller_suite_test.go index f0a981651d..57e5471d03 100644 --- a/pkg/controller/controller_suite_test.go +++ b/pkg/controller/controller_suite_test.go @@ -26,12 +26,12 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics" crscheme "sigs.k8s.io/controller-runtime/pkg/scheme" ) @@ -79,12 +79,12 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) // Prevent the metrics listener being created - metrics.DefaultBindAddress = "0" + metricsserver.DefaultBindAddress = "0" }) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) // Put the DefaultBindAddress back - metrics.DefaultBindAddress = ":8080" + metricsserver.DefaultBindAddress = ":8080" }) diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index b75ed1a4e4..06138a476b 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -24,10 +24,12 @@ import ( . "github.com/onsi/gomega" "go.uber.org/goleak" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/pointer" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" internalcontroller "sigs.k8s.io/controller-runtime/pkg/internal/controller" @@ -59,6 +61,68 @@ var _ = Describe("controller.Controller", func() { Expect(err.Error()).To(ContainSubstring("must specify Reconciler")) }) + It("should return an error if two controllers are registered with the same name", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c1, err := controller.New("c3", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c3", m, controller.Options{Reconciler: rec}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("controller with name c3 already exists")) + Expect(c2).To(BeNil()) + }) + + It("should return an error if two controllers are registered with the same name and SkipNameValidation is set to false on the manager", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ + SkipNameValidation: ptr.To(false), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + c1, err := controller.New("c4", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c4", m, controller.Options{Reconciler: rec}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("controller with name c4 already exists")) + Expect(c2).To(BeNil()) + }) + + It("should not return an error if two controllers are registered with the same name and SkipNameValidation is set on the manager", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ + SkipNameValidation: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + + c1, err := controller.New("c5", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c5", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c2).ToNot(BeNil()) + }) + + It("should not return an error if two controllers are registered with the same name and SkipNameValidation is set on the controller", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c1, err := controller.New("c6", m, controller.Options{Reconciler: rec}) + Expect(err).NotTo(HaveOccurred()) + Expect(c1).ToNot(BeNil()) + + c2, err := controller.New("c6", m, controller.Options{Reconciler: rec, SkipNameValidation: ptr.To(true)}) + Expect(err).NotTo(HaveOccurred()) + Expect(c2).ToNot(BeNil()) + }) + It("should not return an error if two controllers are registered with different names", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -72,12 +136,12 @@ var _ = Describe("controller.Controller", func() { Expect(c2).ToNot(BeNil()) }) - It("should not leak goroutines when stopped", func() { + It("should not leak goroutines when stopped", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) watchChan := make(chan event.GenericEvent, 1) - watch := &source.Channel{Source: watchChan} + watch := source.Channel(watchChan, &handler.EnqueueRequestForObject{}) watchChan <- event.GenericEvent{Object: &corev1.Pod{}} reconcileStarted := make(chan struct{}) @@ -98,8 +162,8 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{Reconciler: rec}) - Expect(c.Watch(watch, &handler.EnqueueRequestForObject{})).To(Succeed()) + c, err := controller.New("new-controller-0", m, controller.Options{Reconciler: rec}) + Expect(c.Watch(watch)).To(Succeed()) Expect(err).NotTo(HaveOccurred()) go func() { @@ -124,7 +188,7 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - _, err = controller.New("new-controller", m, controller.Options{Reconciler: rec}) + _, err = controller.New("new-controller-1", m, controller.Options{Reconciler: rec}) Expect(err).NotTo(HaveOccurred()) // force-close keep-alive connections. These'll time anyway (after @@ -133,16 +197,58 @@ var _ = Describe("controller.Controller", func() { Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) }) + It("should default RateLimiter and NewQueue if not specified", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-2", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.RateLimiter).NotTo(BeNil()) + Expect(ctrl.NewQueue).NotTo(BeNil()) + }) + + It("should not override RateLimiter and NewQueue if specified", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + customRateLimiter := workqueue.NewTypedItemExponentialFailureRateLimiter[reconcile.Request](5*time.Millisecond, 1000*time.Second) + customNewQueueCalled := false + customNewQueue := func(controllerName string, rateLimiter workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + customNewQueueCalled = true + return nil + } + + c, err := controller.New("new-controller-3", m, controller.Options{ + Reconciler: reconcile.Func(nil), + RateLimiter: customRateLimiter, + NewQueue: customNewQueue, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.RateLimiter).To(BeIdenticalTo(customRateLimiter)) + ctrl.NewQueue("controller1", nil) + Expect(customNewQueueCalled).To(BeTrue(), "Expected customNewQueue to be called") + }) + It("should default RecoverPanic from the manager", func() { - m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: pointer.Bool(true)}}) + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: ptr.To(true)}}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-4", m, controller.Options{ Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.RecoverPanic).NotTo(BeNil()) @@ -150,16 +256,16 @@ var _ = Describe("controller.Controller", func() { }) It("should not override RecoverPanic on the controller", func() { - m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: pointer.Bool(true)}}) + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: ptr.To(true)}}) Expect(err).NotTo(HaveOccurred()) c, err := controller.New("new-controller", m, controller.Options{ - RecoverPanic: pointer.Bool(false), + RecoverPanic: ptr.To(false), Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.RecoverPanic).NotTo(BeNil()) @@ -167,31 +273,31 @@ var _ = Describe("controller.Controller", func() { }) It("should default NeedLeaderElection from the manager", func() { - m, err := manager.New(cfg, manager.Options{Controller: config.Controller{NeedLeaderElection: pointer.Bool(true)}}) + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{NeedLeaderElection: ptr.To(true)}}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-5", m, controller.Options{ Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.NeedLeaderElection()).To(BeTrue()) }) It("should not override NeedLeaderElection on the controller", func() { - m, err := manager.New(cfg, manager.Options{Controller: config.Controller{NeedLeaderElection: pointer.Bool(true)}}) + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{NeedLeaderElection: ptr.To(true)}}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ - NeedLeaderElection: pointer.Bool(false), + c, err := controller.New("new-controller-6", m, controller.Options{ + NeedLeaderElection: ptr.To(false), Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.NeedLeaderElection()).To(BeFalse()) @@ -201,12 +307,12 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{Controller: config.Controller{MaxConcurrentReconciles: 5}}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-7", m, controller.Options{ Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.MaxConcurrentReconciles).To(BeEquivalentTo(5)) @@ -216,12 +322,12 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-8", m, controller.Options{ Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.MaxConcurrentReconciles).To(BeEquivalentTo(1)) @@ -231,13 +337,13 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-9", m, controller.Options{ Reconciler: reconcile.Func(nil), MaxConcurrentReconciles: 5, }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.MaxConcurrentReconciles).To(BeEquivalentTo(5)) @@ -247,12 +353,12 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{Controller: config.Controller{CacheSyncTimeout: 5}}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-10", m, controller.Options{ Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.CacheSyncTimeout).To(BeEquivalentTo(5)) @@ -262,12 +368,12 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-11", m, controller.Options{ Reconciler: reconcile.Func(nil), }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.CacheSyncTimeout).To(BeEquivalentTo(2 * time.Minute)) @@ -277,13 +383,13 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-12", m, controller.Options{ Reconciler: reconcile.Func(nil), CacheSyncTimeout: 5, }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.CacheSyncTimeout).To(BeEquivalentTo(5)) @@ -293,12 +399,12 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-13", m, controller.Options{ Reconciler: rec, }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.NeedLeaderElection()).To(BeTrue()) @@ -308,13 +414,13 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ - NeedLeaderElection: pointer.Bool(false), + c, err := controller.New("new-controller-14", m, controller.Options{ + NeedLeaderElection: ptr.To(false), Reconciler: rec, }) Expect(err).NotTo(HaveOccurred()) - ctrl, ok := c.(*internalcontroller.Controller) + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) Expect(ok).To(BeTrue()) Expect(ctrl.NeedLeaderElection()).To(BeFalse()) @@ -324,7 +430,7 @@ var _ = Describe("controller.Controller", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) - c, err := controller.New("new-controller", m, controller.Options{ + c, err := controller.New("new-controller-15", m, controller.Options{ Reconciler: rec, }) Expect(err).NotTo(HaveOccurred()) @@ -332,5 +438,143 @@ var _ = Describe("controller.Controller", func() { _, ok := c.(manager.LeaderElectionRunnable) Expect(ok).To(BeTrue()) }) + + It("should configure a priority queue per default", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{}, + }) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-16", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + q := ctrl.NewQueue("foo", nil) + _, ok = q.(priorityqueue.PriorityQueue[reconcile.Request]) + Expect(ok).To(BeTrue()) + }) + + It("should not configure a priority queue if UsePriorityQueue is set to false", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller-17", m, controller.Options{ + Reconciler: rec, + UsePriorityQueue: ptr.To(false), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + q := ctrl.NewQueue("foo", nil) + _, ok = q.(priorityqueue.PriorityQueue[reconcile.Request]) + Expect(ok).To(BeFalse()) + }) + + It("should set EnableWarmup correctly", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + // Test with EnableWarmup set to true + ctrlWithWarmup, err := controller.New("warmup-enabled-ctrl", m, controller.Options{ + Reconciler: reconcile.Func(nil), + EnableWarmup: ptr.To(true), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlWithWarmup, ok := ctrlWithWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlWithWarmup.EnableWarmup).To(HaveValue(BeTrue())) + + // Test with EnableWarmup set to false + ctrlWithoutWarmup, err := controller.New("warmup-disabled-ctrl", m, controller.Options{ + Reconciler: reconcile.Func(nil), + EnableWarmup: ptr.To(false), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlWithoutWarmup, ok := ctrlWithoutWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlWithoutWarmup.EnableWarmup).To(HaveValue(BeFalse())) + + // Test with EnableWarmup not set (should default to nil) + ctrlWithDefaultWarmup, err := controller.New("warmup-default-ctrl", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlWithDefaultWarmup, ok := ctrlWithDefaultWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlWithDefaultWarmup.EnableWarmup).To(BeNil()) + }) + + It("should inherit EnableWarmup from manager config", func() { + // Test with manager default setting EnableWarmup to true + managerWithWarmup, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ + EnableWarmup: ptr.To(true), + }, + }) + Expect(err).NotTo(HaveOccurred()) + ctrlInheritingWarmup, err := controller.New("inherit-warmup-enabled", managerWithWarmup, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlInheritingWarmup, ok := ctrlInheritingWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlInheritingWarmup.EnableWarmup).To(HaveValue(BeTrue())) + + // Test that explicit controller setting overrides manager setting + ctrlOverridingWarmup, err := controller.New("override-warmup-disabled", managerWithWarmup, controller.Options{ + Reconciler: reconcile.Func(nil), + EnableWarmup: ptr.To(false), + }) + Expect(err).NotTo(HaveOccurred()) + + internalCtrlOverridingWarmup, ok := ctrlOverridingWarmup.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + Expect(internalCtrlOverridingWarmup.EnableWarmup).To(HaveValue(BeFalse())) + }) + + It("should default ReconciliationTimeout from manager if unset", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ReconciliationTimeout: 30 * time.Second}, + }) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("mgr-reconciliation-timeout", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.ReconciliationTimeout).To(Equal(30 * time.Second)) + }) + + It("should not override an existing ReconciliationTimeout", func() { + m, err := manager.New(cfg, manager.Options{ + Controller: config.Controller{ReconciliationTimeout: 30 * time.Second}, + }) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("ctrl-reconciliation-timeout", m, controller.Options{ + Reconciler: rec, + ReconciliationTimeout: time.Minute, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller[reconcile.Request]) + Expect(ok).To(BeTrue()) + + Expect(ctrl.ReconciliationTimeout).To(Equal(time.Minute)) + }) }) }) diff --git a/pkg/controller/controllertest/testing.go b/pkg/controller/controllertest/testing.go index 627915f94b..2b481d2116 100644 --- a/pkg/controller/controllertest/testing.go +++ b/pkg/controller/controllertest/testing.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var _ runtime.Object = &ErrorType{} @@ -36,23 +37,27 @@ func (ErrorType) GetObjectKind() schema.ObjectKind { return nil } // DeepCopyObject implements runtime.Object. func (ErrorType) DeepCopyObject() runtime.Object { return nil } -var _ workqueue.RateLimitingInterface = &Queue{} +var _ workqueue.TypedRateLimitingInterface[reconcile.Request] = &Queue{} // Queue implements a RateLimiting queue as a non-ratelimited queue for testing. // This helps testing by having functions that use a RateLimiting queue synchronously add items to the queue. -type Queue struct { - workqueue.Interface +type Queue = TypedQueue[reconcile.Request] + +// TypedQueue implements a RateLimiting queue as a non-ratelimited queue for testing. +// This helps testing by having functions that use a RateLimiting queue synchronously add items to the queue. +type TypedQueue[request comparable] struct { + workqueue.TypedInterface[request] AddedRateLimitedLock sync.Mutex AddedRatelimited []any } // AddAfter implements RateLimitingInterface. -func (q *Queue) AddAfter(item interface{}, duration time.Duration) { +func (q *TypedQueue[request]) AddAfter(item request, duration time.Duration) { q.Add(item) } // AddRateLimited implements RateLimitingInterface. TODO(community): Implement this. -func (q *Queue) AddRateLimited(item interface{}) { +func (q *TypedQueue[request]) AddRateLimited(item request) { q.AddedRateLimitedLock.Lock() q.AddedRatelimited = append(q.AddedRatelimited, item) q.AddedRateLimitedLock.Unlock() @@ -60,9 +65,9 @@ func (q *Queue) AddRateLimited(item interface{}) { } // Forget implements RateLimitingInterface. TODO(community): Implement this. -func (q *Queue) Forget(item interface{}) {} +func (q *TypedQueue[request]) Forget(item request) {} // NumRequeues implements RateLimitingInterface. TODO(community): Implement this. -func (q *Queue) NumRequeues(item interface{}) int { +func (q *TypedQueue[request]) NumRequeues(item request) int { return 0 } diff --git a/pkg/controller/controllertest/util.go b/pkg/controller/controllertest/util.go index 60ec61edec..2c9a248899 100644 --- a/pkg/controller/controllertest/util.go +++ b/pkg/controller/controllertest/util.go @@ -17,6 +17,7 @@ limitations under the License. package controllertest import ( + "context" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,49 +34,7 @@ type FakeInformer struct { // RunCount is incremented each time RunInformersAndControllers is called RunCount int - handlers []eventHandlerWrapper -} - -type modernResourceEventHandler interface { - OnAdd(obj interface{}, isInInitialList bool) - OnUpdate(oldObj, newObj interface{}) - OnDelete(obj interface{}) -} - -type legacyResourceEventHandler interface { - OnAdd(obj interface{}) - OnUpdate(oldObj, newObj interface{}) - OnDelete(obj interface{}) -} - -// eventHandlerWrapper wraps a ResourceEventHandler in a manner that is compatible with client-go 1.27+ and older. -// The interface was changed in these versions. -type eventHandlerWrapper struct { - handler any -} - -func (e eventHandlerWrapper) OnAdd(obj interface{}) { - if m, ok := e.handler.(modernResourceEventHandler); ok { - m.OnAdd(obj, false) - return - } - e.handler.(legacyResourceEventHandler).OnAdd(obj) -} - -func (e eventHandlerWrapper) OnUpdate(oldObj, newObj interface{}) { - if m, ok := e.handler.(modernResourceEventHandler); ok { - m.OnUpdate(oldObj, newObj) - return - } - e.handler.(legacyResourceEventHandler).OnUpdate(oldObj, newObj) -} - -func (e eventHandlerWrapper) OnDelete(obj interface{}) { - if m, ok := e.handler.(modernResourceEventHandler); ok { - m.OnDelete(obj) - return - } - e.handler.(legacyResourceEventHandler).OnDelete(obj) + handlers []cache.ResourceEventHandler } // AddIndexers does nothing. TODO(community): Implement this. @@ -98,9 +57,21 @@ func (f *FakeInformer) HasSynced() bool { return f.Synced } -// AddEventHandler implements the Informer interface. Adds an EventHandler to the fake Informers. TODO(community): Implement Registration. +// AddEventHandler implements the Informer interface. Adds an EventHandler to the fake Informers. TODO(community): Implement Registration. func (f *FakeInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) { - f.handlers = append(f.handlers, eventHandlerWrapper{handler}) + f.handlers = append(f.handlers, handler) + return nil, nil +} + +// AddEventHandlerWithResyncPeriod implements the Informer interface. Adds an EventHandler to the fake Informers (ignores resyncPeriod). TODO(community): Implement Registration. +func (f *FakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, _ time.Duration) (cache.ResourceEventHandlerRegistration, error) { + f.handlers = append(f.handlers, handler) + return nil, nil +} + +// AddEventHandlerWithOptions implements the Informer interface. Adds an EventHandler to the fake Informers (ignores options). TODO(community): Implement Registration. +func (f *FakeInformer) AddEventHandlerWithOptions(handler cache.ResourceEventHandler, _ cache.HandlerOptions) (cache.ResourceEventHandlerRegistration, error) { + f.handlers = append(f.handlers, handler) return nil, nil } @@ -109,10 +80,14 @@ func (f *FakeInformer) Run(<-chan struct{}) { f.RunCount++ } +func (f *FakeInformer) RunWithContext(_ context.Context) { + f.RunCount++ +} + // Add fakes an Add event for obj. func (f *FakeInformer) Add(obj metav1.Object) { for _, h := range f.handlers { - h.OnAdd(obj) + h.OnAdd(obj, false) } } @@ -130,11 +105,6 @@ func (f *FakeInformer) Delete(obj metav1.Object) { } } -// AddEventHandlerWithResyncPeriod does nothing. TODO(community): Implement this. -func (f *FakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) (cache.ResourceEventHandlerRegistration, error) { - return nil, nil -} - // RemoveEventHandler does nothing. TODO(community): Implement this. func (f *FakeInformer) RemoveEventHandler(handle cache.ResourceEventHandlerRegistration) error { return nil @@ -160,6 +130,11 @@ func (f *FakeInformer) SetWatchErrorHandler(cache.WatchErrorHandler) error { return nil } +// SetWatchErrorHandlerWithContext does nothing. TODO(community): Implement this. +func (f *FakeInformer) SetWatchErrorHandlerWithContext(cache.WatchErrorHandlerWithContext) error { + return nil +} + // SetTransform does nothing. TODO(community): Implement this. func (f *FakeInformer) SetTransform(t cache.TransformFunc) error { return nil diff --git a/pkg/controller/controllerutil/controllerutil.go b/pkg/controller/controllerutil/controllerutil.go index 575aad33ce..0f12b934ee 100644 --- a/pkg/controller/controllerutil/controllerutil.go +++ b/pkg/controller/controllerutil/controllerutil.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "slices" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,7 +28,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) @@ -51,12 +53,22 @@ func newAlreadyOwnedError(obj metav1.Object, owner metav1.OwnerReference) *Alrea } } +// OwnerReferenceOption is a function that can modify a `metav1.OwnerReference`. +type OwnerReferenceOption func(*metav1.OwnerReference) + +// WithBlockOwnerDeletion allows configuring the BlockOwnerDeletion field on the `metav1.OwnerReference`. +func WithBlockOwnerDeletion(blockOwnerDeletion bool) OwnerReferenceOption { + return func(ref *metav1.OwnerReference) { + ref.BlockOwnerDeletion = &blockOwnerDeletion + } +} + // SetControllerReference sets owner as a Controller OwnerReference on controlled. // This is used for garbage collection of the controlled object and for // reconciling the owner object on changes to controlled (with a Watch + EnqueueRequestForOwner). // Since only one OwnerReference can be a controller, it returns an error if // there is another OwnerReference with Controller flag set. -func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme) error { +func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Scheme, opts ...OwnerReferenceOption) error { // Validate the owner. ro, ok := owner.(runtime.Object) if !ok { @@ -76,8 +88,11 @@ func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Sch Kind: gvk.Kind, Name: owner.GetName(), UID: owner.GetUID(), - BlockOwnerDeletion: pointer.Bool(true), - Controller: pointer.Bool(true), + BlockOwnerDeletion: ptr.To(true), + Controller: ptr.To(true), + } + for _, opt := range opts { + opt(&ref) } // Return early with an error if the object is already controlled. @@ -93,7 +108,7 @@ func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Sch // SetOwnerReference is a helper method to make sure the given object contains an object reference to the object provided. // This allows you to declare that owner has a dependency on the object without specifying it as a controller. // If a reference to the same object already exists, it'll be overwritten with the newly provided version. -func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) error { +func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme, opts ...OwnerReferenceOption) error { // Validate the owner. ro, ok := owner.(runtime.Object) if !ok { @@ -114,12 +129,108 @@ func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) erro UID: owner.GetUID(), Name: owner.GetName(), } + for _, opt := range opts { + opt(&ref) + } // Update owner references and return. upsertOwnerRef(ref, object) return nil } +// RemoveOwnerReference is a helper method to make sure the given object removes an owner reference to the object provided. +// This allows you to remove the owner to establish a new owner of the object in a subsequent call. +func RemoveOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) error { + owners := object.GetOwnerReferences() + length := len(owners) + if length < 1 { + return fmt.Errorf("%T does not have any owner references", object) + } + ro, ok := owner.(runtime.Object) + if !ok { + return fmt.Errorf("%T is not a runtime.Object, cannot call RemoveOwnerReference", owner) + } + gvk, err := apiutil.GVKForObject(ro, scheme) + if err != nil { + return err + } + + index := indexOwnerRef(owners, metav1.OwnerReference{ + APIVersion: gvk.GroupVersion().String(), + Name: owner.GetName(), + Kind: gvk.Kind, + }) + if index == -1 { + return fmt.Errorf("%T does not have an owner reference for %T", object, owner) + } + + owners = append(owners[:index], owners[index+1:]...) + object.SetOwnerReferences(owners) + return nil +} + +// HasControllerReference returns true if the object +// has an owner ref with controller equal to true +func HasControllerReference(object metav1.Object) bool { + owners := object.GetOwnerReferences() + for _, owner := range owners { + isTrue := owner.Controller + if owner.Controller != nil && *isTrue { + return true + } + } + return false +} + +// HasOwnerReference returns true if the owners list contains an owner reference +// that matches the object's group, kind, and name. +func HasOwnerReference(ownerRefs []metav1.OwnerReference, obj client.Object, scheme *runtime.Scheme) (bool, error) { + gvk, err := apiutil.GVKForObject(obj, scheme) + if err != nil { + return false, err + } + idx := indexOwnerRef(ownerRefs, metav1.OwnerReference{ + APIVersion: gvk.GroupVersion().String(), + Name: obj.GetName(), + Kind: gvk.Kind, + }) + return idx != -1, nil +} + +// RemoveControllerReference removes an owner reference where the controller +// equals true +func RemoveControllerReference(owner, object metav1.Object, scheme *runtime.Scheme) error { + if ok := HasControllerReference(object); !ok { + return fmt.Errorf("%T does not have a owner reference with controller equals true", object) + } + ro, ok := owner.(runtime.Object) + if !ok { + return fmt.Errorf("%T is not a runtime.Object, cannot call RemoveControllerReference", owner) + } + gvk, err := apiutil.GVKForObject(ro, scheme) + if err != nil { + return err + } + ownerRefs := object.GetOwnerReferences() + index := indexOwnerRef(ownerRefs, metav1.OwnerReference{ + APIVersion: gvk.GroupVersion().String(), + Name: owner.GetName(), + Kind: gvk.Kind, + }) + + if index == -1 { + return fmt.Errorf("%T does not have an controller reference for %T", object, owner) + } + + if ownerRefs[index].Controller == nil || !*ownerRefs[index].Controller { + return fmt.Errorf("%T owner is not the controller reference for %T", owner, object) + } + + ownerRefs = append(ownerRefs[:index], ownerRefs[index+1:]...) + object.SetOwnerReferences(ownerRefs) + return nil +} + func upsertOwnerRef(ref metav1.OwnerReference, object metav1.Object) { owners := object.GetOwnerReferences() if idx := indexOwnerRef(owners, ref); idx == -1 { @@ -165,11 +276,10 @@ func referSameObject(a, b metav1.OwnerReference) bool { if err != nil { return false } - return aGV.Group == bGV.Group && a.Kind == b.Kind && a.Name == b.Name } -// OperationResult is the action result of a CreateOrUpdate call. +// OperationResult is the action result of a CreateOrUpdate or CreateOrPatch call. type OperationResult string const ( // They should complete the sentence "Deployment default/foo has been ..." @@ -185,22 +295,41 @@ const ( // They should complete the sentence "Deployment default/foo has been .. OperationResultUpdatedStatusOnly OperationResult = "updatedStatusOnly" ) -// CreateOrUpdate creates or updates the given object in the Kubernetes -// cluster. The object's desired state must be reconciled with the existing -// state inside the passed in callback MutateFn. +// CreateOrUpdate attempts to fetch the given object from the Kubernetes cluster. +// If the object didn't exist, MutateFn will be called, and it will be created. +// If the object did exist, MutateFn will be called, and if it changed the +// object, it will be updated. +// Otherwise, it will be left unchanged. +// The executed operation (and an error) will be returned. // -// The MutateFn is called regardless of creating or updating an object. +// WARNING: If the MutateFn resets a value on obj that has a default value, +// CreateOrUpdate will *always* perform an update. This is because when the +// object is fetched from the API server, the value will have taken on the +// default value, and the check for equality will fail. For example, Deployments +// must have a Replicas value set. If the MutateFn sets a Deployment's Replicas +// to nil, then it will never match with the object returned from the API +// server, which defaults the value to 1. // -// It returns the executed operation and an error. +// WARNING: CreateOrUpdate assumes that no values have been set on obj aside +// from the Name/Namespace. Values other than Name and Namespace that existed on +// obj may be overwritten by the corresponding values in the object returned +// from the Kubernetes API server. When this happens, the Update will not work +// as expected. +// +// Note: changes made by MutateFn to any sub-resource (status...), will be +// discarded. func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (OperationResult, error) { key := client.ObjectKeyFromObject(obj) if err := c.Get(ctx, key, obj); err != nil { if !apierrors.IsNotFound(err) { return OperationResultNone, err } - if err := mutate(f, key, obj); err != nil { - return OperationResultNone, err + if f != nil { + if err := mutate(f, key, obj); err != nil { + return OperationResultNone, err + } } + if err := c.Create(ctx, obj); err != nil { return OperationResultNone, err } @@ -208,8 +337,10 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f M } existing := obj.DeepCopyObject() - if err := mutate(f, key, obj); err != nil { - return OperationResultNone, err + if f != nil { + if err := mutate(f, key, obj); err != nil { + return OperationResultNone, err + } } if equality.Semantic.DeepEqual(existing, obj) { @@ -222,13 +353,32 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f M return OperationResultUpdated, nil } -// CreateOrPatch creates or patches the given object in the Kubernetes -// cluster. The object's desired state must be reconciled with the before -// state inside the passed in callback MutateFn. +// CreateOrPatch attempts to fetch the given object from the Kubernetes cluster. +// If the object didn't exist, MutateFn will be called, and it will be created. +// If the object did exist, MutateFn will be called, and if it changed the +// object, it will be patched. +// Otherwise, it will be left unchanged. +// The executed operation (and an error) will be returned. +// +// WARNING: If the MutateFn resets a value on obj that has a default value, +// CreateOrPatch will *always* perform a patch. This is because when the +// object is fetched from the API server, the value will have taken on the +// default value, and the check for equality will fail. +// For example, Deployments must have a Replicas value set. If the MutateFn sets +// a Deployment's Replicas to nil, then it will never match with the object +// returned from the API server, which defaults the value to 1. // -// The MutateFn is called regardless of creating or updating an object. +// WARNING: CreateOrPatch assumes that no values have been set on obj aside +// from the Name/Namespace. Values other than Name and Namespace that existed on +// obj may be overwritten by the corresponding values in the object returned +// from the Kubernetes API server. When this happens, the Patch will not work +// as expected. // -// It returns the executed operation and an error. +// Note: changes to any sub-resource other than status will be ignored. +// Changes to the status sub-resource will only be applied if the object +// already exist. To change the status on object creation, the easiest +// way is to requeue the object in the controller if OperationResult is +// OperationResultCreated func CreateOrPatch(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (OperationResult, error) { key := client.ObjectKeyFromObject(obj) if err := c.Get(ctx, key, obj); err != nil { @@ -352,10 +502,8 @@ type MutateFn func() error // It returns an indication of whether it updated the object's list of finalizers. func AddFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) { f := o.GetFinalizers() - for _, e := range f { - if e == finalizer { - return false - } + if slices.Contains(f, finalizer) { + return false } o.SetFinalizers(append(f, finalizer)) return true @@ -368,7 +516,7 @@ func RemoveFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) length := len(f) index := 0 - for i := 0; i < length; i++ { + for i := range length { if f[i] == finalizer { continue } @@ -382,16 +530,5 @@ func RemoveFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) // ContainsFinalizer checks an Object that the provided finalizer is present. func ContainsFinalizer(o client.Object, finalizer string) bool { f := o.GetFinalizers() - for _, e := range f { - if e == finalizer { - return true - } - } - return false + return slices.Contains(f, finalizer) } - -// Object allows functions to work indistinctly with any resource that -// implements both Object interfaces. -// -// Deprecated: Use client.Object instead. -type Object = client.Object diff --git a/pkg/controller/controllerutil/controllerutil_test.go b/pkg/controller/controllerutil/controllerutil_test.go index a058ce0714..a716667f6a 100644 --- a/pkg/controller/controllerutil/controllerutil_test.go +++ b/pkg/controller/controllerutil/controllerutil_test.go @@ -30,7 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -51,6 +51,23 @@ var _ = Describe("Controllerutil", func() { })) }) + It("should set the BlockOwnerDeletion if it is specified as an option", func() { + t := true + rs := &appsv1.ReplicaSet{} + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme, controllerutil.WithBlockOwnerDeletion(true))).ToNot(HaveOccurred()) + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid", + BlockOwnerDeletion: &t, + })) + }) + It("should not duplicate owner references", func() { rs := &appsv1.ReplicaSet{ ObjectMeta: metav1.ObjectMeta{ @@ -102,6 +119,161 @@ var _ = Describe("Controllerutil", func() { UID: "foo-uid-2", })) }) + It("should remove the owner reference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + { + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1alpha1", + UID: "foo-uid-1", + }, + }, + }, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid-2", + })) + Expect(controllerutil.RemoveOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) + }) + It("should remove the owner reference established by the SetControllerReference function", func() { + rs := &appsv1.ReplicaSet{} + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).NotTo(HaveOccurred()) + t := true + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid", + Controller: &t, + BlockOwnerDeletion: &t, + })) + Expect(controllerutil.RemoveOwnerReference(dep, rs, scheme.Scheme)).NotTo(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) + }) + It("should error when trying to remove the reference that doesn't exist", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.RemoveOwnerReference(dep, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) + }) + It("should error when trying to remove the reference that doesn't abide by the scheme", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveOwnerReference(dep, rs, runtime.NewScheme())).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(1)) + }) + It("should error when trying to remove the owner when setting the owner as a non runtime.Object", func() { + var obj metav1.Object + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveOwnerReference(obj, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(1)) + }) + + It("should error when trying to remove an owner that doesn't exist", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + dep2 := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "bar", UID: "bar-uid-3"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveOwnerReference(dep2, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(1)) + }) + + It("should return true when HasControllerReference evaluates owner reference set by SetControllerReference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.HasControllerReference(rs)).To(BeTrue()) + }) + + It("should return false when HasControllerReference evaluates owner reference set by SetOwnerReference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.HasControllerReference(rs)).To(BeFalse()) + }) + + It("should error when RemoveControllerReference owner's controller is set to false", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveControllerReference(dep, rs, scheme.Scheme)).To(HaveOccurred()) + }) + + It("should error when RemoveControllerReference passed in owner is not the owner", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + dep2 := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo-2", UID: "foo-uid-42"}, + } + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.SetOwnerReference(dep2, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveControllerReference(dep2, rs, scheme.Scheme)).To(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(HaveLen(2)) + }) + + It("should not error when RemoveControllerReference owner's controller is set to true", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid-2"}, + } + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(controllerutil.RemoveControllerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + Expect(rs.GetOwnerReferences()).To(BeEmpty()) + }) }) Describe("SetControllerReference", func() { @@ -255,6 +427,25 @@ var _ = Describe("Controllerutil", func() { BlockOwnerDeletion: &t, })) }) + + It("should set the BlockOwnerDeletion if it is specified as an option", func() { + f := false + t := true + rs := &appsv1.ReplicaSet{} + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + + Expect(controllerutil.SetControllerReference(dep, rs, scheme.Scheme, controllerutil.WithBlockOwnerDeletion(false))).NotTo(HaveOccurred()) + Expect(rs.OwnerReferences).To(ConsistOf(metav1.OwnerReference{ + Name: "foo", + Kind: "Deployment", + APIVersion: "extensions/v1beta1", + UID: "foo-uid", + Controller: &t, + BlockOwnerDeletion: &f, + })) + }) }) Describe("CreateOrUpdate", func() { @@ -266,7 +457,7 @@ var _ = Describe("Controllerutil", func() { BeforeEach(func() { deploy = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("deploy-%d", rand.Int31()), //nolint:gosec + Name: fmt.Sprintf("deploy-%d", rand.Int31()), Namespace: "default", }, } @@ -300,8 +491,8 @@ var _ = Describe("Controllerutil", func() { specr = deploymentSpecr(deploy, deplSpec) }) - It("creates a new object if one doesn't exists", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("creates a new object if one doesn't exists", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -311,7 +502,7 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment created") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) By("being mutated by MutateFn") Expect(fetched.Spec.Template.Spec.Containers).To(HaveLen(1)) @@ -319,13 +510,13 @@ var _ = Describe("Controllerutil", func() { Expect(fetched.Spec.Template.Spec.Containers[0].Image).To(Equal(deplSpec.Template.Spec.Containers[0].Image)) }) - It("updates existing object", func() { + It("updates existing object", func(ctx SpecContext) { var scale int32 = 2 - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(err).NotTo(HaveOccurred()) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentScaler(deploy, scale)) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentScaler(deploy, scale)) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -334,17 +525,17 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment scaled") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) Expect(*fetched.Spec.Replicas).To(Equal(scale)) }) - It("updates only changed objects", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("updates only changed objects", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentIdentity) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentIdentity) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -352,8 +543,8 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when MutateFn changes object name on creation", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, func() error { + It("errors when MutateFn changes object name on creation", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, func() error { Expect(specr()).To(Succeed()) return deploymentRenamer(deploy)() }) @@ -365,13 +556,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when MutateFn renames an object", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("errors when MutateFn renames an object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentRenamer(deploy)) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentRenamer(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -380,13 +571,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when object namespace changes", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), c, deploy, specr) + It("errors when object namespace changes", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrUpdate(context.TODO(), c, deploy, deploymentNamespaceChanger(deploy)) + op, err = controllerutil.CreateOrUpdate(ctx, c, deploy, deploymentNamespaceChanger(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -395,8 +586,8 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("aborts immediately if there was an error initially retrieving the object", func() { - op, err := controllerutil.CreateOrUpdate(context.TODO(), errorReader{c}, deploy, func() error { + It("aborts immediately if there was an error initially retrieving the object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrUpdate(ctx, errorReader{c}, deploy, func() error { Fail("Mutation method should not run") return nil }) @@ -415,7 +606,7 @@ var _ = Describe("Controllerutil", func() { BeforeEach(func() { deploy = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("deploy-%d", rand.Int31()), //nolint:gosec + Name: fmt.Sprintf("deploy-%d", rand.Int31()), Namespace: "default", }, } @@ -449,22 +640,22 @@ var _ = Describe("Controllerutil", func() { specr = deploymentSpecr(deploy, deplSpec) }) - assertLocalDeployWasUpdated := func(fetched *appsv1.Deployment) { + assertLocalDeployWasUpdated := func(ctx context.Context, fetched *appsv1.Deployment) { By("local deploy object was updated during patch & has same spec, status, resource version as fetched") if fetched == nil { fetched = &appsv1.Deployment{} - ExpectWithOffset(1, c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + ExpectWithOffset(1, c.Get(ctx, deplKey, fetched)).To(Succeed()) } ExpectWithOffset(1, fetched.ResourceVersion).To(Equal(deploy.ResourceVersion)) ExpectWithOffset(1, fetched.Spec).To(BeEquivalentTo(deploy.Spec)) ExpectWithOffset(1, fetched.Status).To(BeEquivalentTo(deploy.Status)) } - assertLocalDeployStatusWasUpdated := func(fetched *appsv1.Deployment) { + assertLocalDeployStatusWasUpdated := func(ctx context.Context, fetched *appsv1.Deployment) { By("local deploy object was updated during patch & has same spec, status, resource version as fetched") if fetched == nil { fetched = &appsv1.Deployment{} - ExpectWithOffset(1, c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + ExpectWithOffset(1, c.Get(ctx, deplKey, fetched)).To(Succeed()) } ExpectWithOffset(1, fetched.ResourceVersion).To(Equal(deploy.ResourceVersion)) ExpectWithOffset(1, *fetched.Spec.Replicas).To(BeEquivalentTo(int32(5))) @@ -472,8 +663,8 @@ var _ = Describe("Controllerutil", func() { ExpectWithOffset(1, len(fetched.Status.Conditions)).To(BeEquivalentTo(1)) } - It("creates a new object if one doesn't exists", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("creates a new object if one doesn't exists", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -483,7 +674,7 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment created") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) By("being mutated by MutateFn") Expect(fetched.Spec.Template.Spec.Containers).To(HaveLen(1)) @@ -491,13 +682,13 @@ var _ = Describe("Controllerutil", func() { Expect(fetched.Spec.Template.Spec.Containers[0].Image).To(Equal(deplSpec.Template.Spec.Containers[0].Image)) }) - It("patches existing object", func() { + It("patches existing object", func(ctx SpecContext) { var scale int32 = 2 - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(err).NotTo(HaveOccurred()) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentScaler(deploy, scale)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentScaler(deploy, scale)) By("returning no error") Expect(err).NotTo(HaveOccurred()) @@ -506,29 +697,29 @@ var _ = Describe("Controllerutil", func() { By("actually having the deployment scaled") fetched := &appsv1.Deployment{} - Expect(c.Get(context.TODO(), deplKey, fetched)).To(Succeed()) + Expect(c.Get(ctx, deplKey, fetched)).To(Succeed()) Expect(*fetched.Spec.Replicas).To(Equal(scale)) - assertLocalDeployWasUpdated(fetched) + assertLocalDeployWasUpdated(ctx, fetched) }) - It("patches only changed objects", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches only changed objects", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentIdentity) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentIdentity) By("returning no error") Expect(err).NotTo(HaveOccurred()) By("returning OperationResultNone") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) }) - It("patches only changed status", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches only changed status", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) @@ -537,18 +728,18 @@ var _ = Describe("Controllerutil", func() { ReadyReplicas: 1, Replicas: 3, } - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentStatusr(deploy, deployStatus)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentStatusr(deploy, deployStatus)) By("returning no error") Expect(err).NotTo(HaveOccurred()) By("returning OperationResultUpdatedStatusOnly") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatusOnly)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) }) - It("patches resource and status", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches resource and status", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) @@ -558,7 +749,7 @@ var _ = Describe("Controllerutil", func() { ReadyReplicas: 1, Replicas: replicas, } - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, func() error { Expect(deploymentScaler(deploy, replicas)()).To(Succeed()) return deploymentStatusr(deploy, deployStatus)() }) @@ -568,11 +759,11 @@ var _ = Describe("Controllerutil", func() { By("returning OperationResultUpdatedStatus") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatus)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) }) - It("patches resource and not empty status", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("patches resource and not empty status", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) @@ -582,7 +773,7 @@ var _ = Describe("Controllerutil", func() { ReadyReplicas: 1, Replicas: replicas, } - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, func() error { Expect(deploymentScaler(deploy, replicas)()).To(Succeed()) return deploymentStatusr(deploy, deployStatus)() }) @@ -592,10 +783,10 @@ var _ = Describe("Controllerutil", func() { By("returning OperationResultUpdatedStatus") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatus)) - assertLocalDeployWasUpdated(nil) + assertLocalDeployWasUpdated(ctx, nil) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { - deploy.Spec.Replicas = pointer.Int32(5) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, func() error { + deploy.Spec.Replicas = ptr.To(int32(5)) deploy.Status.Conditions = []appsv1.DeploymentCondition{{ Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, @@ -608,11 +799,11 @@ var _ = Describe("Controllerutil", func() { By("returning OperationResultUpdatedStatus") Expect(op).To(BeEquivalentTo(controllerutil.OperationResultUpdatedStatus)) - assertLocalDeployStatusWasUpdated(nil) + assertLocalDeployStatusWasUpdated(ctx, nil) }) - It("errors when MutateFn changes object name on creation", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { + It("errors when MutateFn changes object name on creation", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, func() error { Expect(specr()).To(Succeed()) return deploymentRenamer(deploy)() }) @@ -624,13 +815,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when MutateFn renames an object", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("errors when MutateFn renames an object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentRenamer(deploy)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentRenamer(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -639,13 +830,13 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("errors when object namespace changes", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), c, deploy, specr) + It("errors when object namespace changes", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, c, deploy, specr) Expect(op).To(BeEquivalentTo(controllerutil.OperationResultCreated)) Expect(err).NotTo(HaveOccurred()) - op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, deploymentNamespaceChanger(deploy)) + op, err = controllerutil.CreateOrPatch(ctx, c, deploy, deploymentNamespaceChanger(deploy)) By("returning error") Expect(err).To(HaveOccurred()) @@ -654,8 +845,8 @@ var _ = Describe("Controllerutil", func() { Expect(op).To(BeEquivalentTo(controllerutil.OperationResultNone)) }) - It("aborts immediately if there was an error initially retrieving the object", func() { - op, err := controllerutil.CreateOrPatch(context.TODO(), errorReader{c}, deploy, func() error { + It("aborts immediately if there was an error initially retrieving the object", func(ctx SpecContext) { + op, err := controllerutil.CreateOrPatch(ctx, errorReader{c}, deploy, func() error { Fail("Mutation method should not run") return nil }) @@ -766,6 +957,33 @@ var _ = Describe("Controllerutil", func() { Expect(controllerutil.ContainsFinalizer(deploy, testFinalizer)).To(BeFalse()) }) }) + + Describe("HasOwnerReference", func() { + It("should return true if the object has the owner reference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + Expect(controllerutil.SetOwnerReference(dep, rs, scheme.Scheme)).ToNot(HaveOccurred()) + b, err := controllerutil.HasOwnerReference(rs.GetOwnerReferences(), dep, scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + Expect(b).To(BeTrue()) + }) + + It("should return false if the object does not have the owner reference", func() { + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + dep := &extensionsv1beta1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", UID: "foo-uid"}, + } + b, err := controllerutil.HasOwnerReference(rs.GetOwnerReferences(), dep, scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + Expect(b).To(BeFalse()) + }) + }) }) }) diff --git a/pkg/controller/example_test.go b/pkg/controller/example_test.go index d4fa1aef0b..e3c4b6a092 100644 --- a/pkg/controller/example_test.go +++ b/pkg/controller/example_test.go @@ -71,7 +71,7 @@ func ExampleController() { } // Watch for Pod create / update / delete events and call Reconcile - err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{})) if err != nil { log.Error(err, "unable to watch pods") os.Exit(1) @@ -108,7 +108,7 @@ func ExampleController_unstructured() { Version: "v1", }) // Watch for Pod create / update / delete events and call Reconcile - err = c.Watch(source.Kind(mgr.GetCache(), u), &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), u, &handler.TypedEnqueueRequestForObject[*unstructured.Unstructured]{})) if err != nil { log.Error(err, "unable to watch pods") os.Exit(1) @@ -129,7 +129,7 @@ func ExampleNewUnmanaged() { // Configure creates a new controller but does not add it to the supplied // manager. - c, err := controller.NewUnmanaged("pod-controller", mgr, controller.Options{ + c, err := controller.NewUnmanaged("pod-controller", controller.Options{ Reconciler: reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { return reconcile.Result{}, nil }), @@ -139,7 +139,7 @@ func ExampleNewUnmanaged() { os.Exit(1) } - if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.EnqueueRequestForObject{}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{})); err != nil { log.Error(err, "unable to watch pods") os.Exit(1) } diff --git a/pkg/ratelimiter/ratelimiter.go b/pkg/controller/name.go similarity index 50% rename from pkg/ratelimiter/ratelimiter.go rename to pkg/controller/name.go index 565a3a227f..00ca655128 100644 --- a/pkg/ratelimiter/ratelimiter.go +++ b/pkg/controller/name.go @@ -14,17 +14,30 @@ See the License for the specific language governing permissions and limitations under the License. */ -package ratelimiter - -import "time" - -// RateLimiter is an identical interface of client-go workqueue RateLimiter. -type RateLimiter interface { - // When gets an item and gets to decide how long that item should wait - When(item interface{}) time.Duration - // Forget indicates that an item is finished being retried. Doesn't matter whether its for perm failing - // or for success, we'll stop tracking it - Forget(item interface{}) - // NumRequeues returns back how many failures the item has had - NumRequeues(item interface{}) int +package controller + +import ( + "fmt" + "sync" + + "k8s.io/apimachinery/pkg/util/sets" +) + +var nameLock sync.Mutex +var usedNames sets.Set[string] + +func checkName(name string) error { + nameLock.Lock() + defer nameLock.Unlock() + if usedNames == nil { + usedNames = sets.Set[string]{} + } + + if usedNames.Has(name) { + return fmt.Errorf("controller with name %s already exists. Controller names must be unique to avoid multiple controllers reporting the same metric. This validation can be disabled via the SkipNameValidation option", name) + } + + usedNames.Insert(name) + + return nil } diff --git a/pkg/controller/priorityqueue/metrics.go b/pkg/controller/priorityqueue/metrics.go new file mode 100644 index 0000000000..967a252dfb --- /dev/null +++ b/pkg/controller/priorityqueue/metrics.go @@ -0,0 +1,172 @@ +package priorityqueue + +import ( + "sync" + "time" + + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" + "sigs.k8s.io/controller-runtime/pkg/internal/metrics" +) + +// This file is mostly a copy of unexported code from +// https://github.com/kubernetes/kubernetes/blob/1d8828ce707ed9dd7a6a9756385419cce1d202ac/staging/src/k8s.io/client-go/util/workqueue/metrics.go +// +// The only two differences are the addition of mapLock in defaultQueueMetrics and converging retryMetrics into queueMetrics. + +type queueMetrics[T comparable] interface { + add(item T, priority int) + get(item T, priority int) + updateDepthWithPriorityMetric(oldPriority, newPriority int) + done(item T) + updateUnfinishedWork() + retry() +} + +func newQueueMetrics[T comparable](mp workqueue.MetricsProvider, name string, clock clock.Clock) queueMetrics[T] { + if len(name) == 0 { + return noMetrics[T]{} + } + + dqm := &defaultQueueMetrics[T]{ + clock: clock, + adds: mp.NewAddsMetric(name), + latency: mp.NewLatencyMetric(name), + workDuration: mp.NewWorkDurationMetric(name), + unfinishedWorkSeconds: mp.NewUnfinishedWorkSecondsMetric(name), + longestRunningProcessor: mp.NewLongestRunningProcessorSecondsMetric(name), + addTimes: map[T]time.Time{}, + processingStartTimes: map[T]time.Time{}, + retries: mp.NewRetriesMetric(name), + } + + if mpp, ok := mp.(metrics.MetricsProviderWithPriority); ok { + dqm.depthWithPriority = mpp.NewDepthMetricWithPriority(name) + } else { + dqm.depth = mp.NewDepthMetric(name) + } + return dqm +} + +// defaultQueueMetrics expects the caller to lock before setting any metrics. +type defaultQueueMetrics[T comparable] struct { + clock clock.Clock + + // current depth of a workqueue + depth workqueue.GaugeMetric + depthWithPriority metrics.DepthMetricWithPriority + // total number of adds handled by a workqueue + adds workqueue.CounterMetric + // how long an item stays in a workqueue + latency workqueue.HistogramMetric + // how long processing an item from a workqueue takes + workDuration workqueue.HistogramMetric + + mapLock sync.RWMutex + addTimes map[T]time.Time + processingStartTimes map[T]time.Time + + // how long have current threads been working? + unfinishedWorkSeconds workqueue.SettableGaugeMetric + longestRunningProcessor workqueue.SettableGaugeMetric + + retries workqueue.CounterMetric +} + +// add is called for ready items only +func (m *defaultQueueMetrics[T]) add(item T, priority int) { + if m == nil { + return + } + + m.adds.Inc() + if m.depthWithPriority != nil { + m.depthWithPriority.Inc(priority) + } else { + m.depth.Inc() + } + + m.mapLock.Lock() + defer m.mapLock.Unlock() + + if _, exists := m.addTimes[item]; !exists { + m.addTimes[item] = m.clock.Now() + } +} + +func (m *defaultQueueMetrics[T]) get(item T, priority int) { + if m == nil { + return + } + + if m.depthWithPriority != nil { + m.depthWithPriority.Dec(priority) + } else { + m.depth.Dec() + } + + m.mapLock.Lock() + defer m.mapLock.Unlock() + + m.processingStartTimes[item] = m.clock.Now() + if startTime, exists := m.addTimes[item]; exists { + m.latency.Observe(m.sinceInSeconds(startTime)) + delete(m.addTimes, item) + } +} + +func (m *defaultQueueMetrics[T]) updateDepthWithPriorityMetric(oldPriority, newPriority int) { + if m.depthWithPriority != nil { + m.depthWithPriority.Dec(oldPriority) + m.depthWithPriority.Inc(newPriority) + } +} + +func (m *defaultQueueMetrics[T]) done(item T) { + if m == nil { + return + } + + m.mapLock.Lock() + defer m.mapLock.Unlock() + if startTime, exists := m.processingStartTimes[item]; exists { + m.workDuration.Observe(m.sinceInSeconds(startTime)) + delete(m.processingStartTimes, item) + } +} + +func (m *defaultQueueMetrics[T]) updateUnfinishedWork() { + m.mapLock.RLock() + defer m.mapLock.RUnlock() + // Note that a summary metric would be better for this, but prometheus + // doesn't seem to have non-hacky ways to reset the summary metrics. + var total float64 + var oldest float64 + for _, t := range m.processingStartTimes { + age := m.sinceInSeconds(t) + total += age + if age > oldest { + oldest = age + } + } + m.unfinishedWorkSeconds.Set(total) + m.longestRunningProcessor.Set(oldest) +} + +// Gets the time since the specified start in seconds. +func (m *defaultQueueMetrics[T]) sinceInSeconds(start time.Time) float64 { + return m.clock.Since(start).Seconds() +} + +func (m *defaultQueueMetrics[T]) retry() { + m.retries.Inc() +} + +type noMetrics[T any] struct{} + +func (noMetrics[T]) add(item T, priority int) {} +func (noMetrics[T]) get(item T, priority int) {} +func (noMetrics[T]) updateDepthWithPriorityMetric(oldPriority, newPriority int) {} +func (noMetrics[T]) done(item T) {} +func (noMetrics[T]) updateUnfinishedWork() {} +func (noMetrics[T]) retry() {} diff --git a/pkg/controller/priorityqueue/metrics_test.go b/pkg/controller/priorityqueue/metrics_test.go new file mode 100644 index 0000000000..3be3989d89 --- /dev/null +++ b/pkg/controller/priorityqueue/metrics_test.go @@ -0,0 +1,141 @@ +package priorityqueue + +import ( + "sync" + + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/internal/metrics" +) + +func newFakeMetricsProvider() *fakeMetricsProvider { + return &fakeMetricsProvider{ + depth: make(map[string]map[int]int), + adds: make(map[string]int), + latency: make(map[string][]float64), + workDuration: make(map[string][]float64), + unfinishedWorkSeconds: make(map[string]float64), + longestRunningProcessor: make(map[string]float64), + retries: make(map[string]int), + mu: sync.Mutex{}, + } +} + +var _ metrics.MetricsProviderWithPriority = &fakeMetricsProvider{} + +type fakeMetricsProvider struct { + depth map[string]map[int]int + adds map[string]int + latency map[string][]float64 + workDuration map[string][]float64 + unfinishedWorkSeconds map[string]float64 + longestRunningProcessor map[string]float64 + retries map[string]int + mu sync.Mutex +} + +func (f *fakeMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { + panic("Should never be called. Expected NewDepthMetricWithPriority to be called instead") +} + +func (f *fakeMetricsProvider) NewDepthMetricWithPriority(name string) metrics.DepthMetricWithPriority { + f.mu.Lock() + defer f.mu.Unlock() + f.depth[name] = map[int]int{} + return &fakeGaugeMetric{m: &f.depth, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.adds[name] = 0 + return &fakeCounterMetric{m: &f.adds, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.latency[name] = []float64{} + return &fakeHistogramMetric{m: &f.latency, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.workDuration[name] = []float64{} + return &fakeHistogramMetric{m: &f.workDuration, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.unfinishedWorkSeconds[name] = 0 + return &fakeSettableGaugeMetric{m: &f.unfinishedWorkSeconds, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.longestRunningProcessor[name] = 0 + return &fakeSettableGaugeMetric{m: &f.longestRunningProcessor, mu: &f.mu, name: name} +} + +func (f *fakeMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { + f.mu.Lock() + defer f.mu.Unlock() + f.retries[name] = 0 + return &fakeCounterMetric{m: &f.retries, mu: &f.mu, name: name} +} + +type fakeGaugeMetric struct { + m *map[string]map[int]int + mu *sync.Mutex + name string +} + +func (fg *fakeGaugeMetric) Inc(priority int) { + fg.mu.Lock() + defer fg.mu.Unlock() + (*fg.m)[fg.name][priority]++ +} + +func (fg *fakeGaugeMetric) Dec(priority int) { + fg.mu.Lock() + defer fg.mu.Unlock() + (*fg.m)[fg.name][priority]-- +} + +type fakeCounterMetric struct { + m *map[string]int + mu *sync.Mutex + name string +} + +func (fc *fakeCounterMetric) Inc() { + fc.mu.Lock() + defer fc.mu.Unlock() + (*fc.m)[fc.name]++ +} + +type fakeHistogramMetric struct { + m *map[string][]float64 + mu *sync.Mutex + name string +} + +func (fh *fakeHistogramMetric) Observe(v float64) { + fh.mu.Lock() + defer fh.mu.Unlock() + (*fh.m)[fh.name] = append((*fh.m)[fh.name], v) +} + +type fakeSettableGaugeMetric struct { + m *map[string]float64 + mu *sync.Mutex + name string +} + +func (fs *fakeSettableGaugeMetric) Set(v float64) { + fs.mu.Lock() + defer fs.mu.Unlock() + (*fs.m)[fs.name] = v +} diff --git a/pkg/controller/priorityqueue/priorityqueue.go b/pkg/controller/priorityqueue/priorityqueue.go new file mode 100644 index 0000000000..98df84c56b --- /dev/null +++ b/pkg/controller/priorityqueue/priorityqueue.go @@ -0,0 +1,460 @@ +package priorityqueue + +import ( + "math" + "sync" + "sync/atomic" + "time" + + "github.com/go-logr/logr" + "github.com/google/btree" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/clock" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/internal/metrics" +) + +// AddOpts describes the options for adding items to the queue. +type AddOpts struct { + After time.Duration + RateLimited bool + // Priority is the priority of the item. Higher values + // indicate higher priority. + // Defaults to zero if unset. + Priority *int +} + +// PriorityQueue is a priority queue for a controller. It +// internally de-duplicates all items that are added to +// it. It will use the max of the passed priorities and the +// min of possible durations. +type PriorityQueue[T comparable] interface { + workqueue.TypedRateLimitingInterface[T] + AddWithOpts(o AddOpts, Items ...T) + GetWithPriority() (item T, priority int, shutdown bool) +} + +// Opts contains the options for a PriorityQueue. +type Opts[T comparable] struct { + // Ratelimiter is being used when AddRateLimited is called. Defaults to a per-item exponential backoff + // limiter with an initial delay of five milliseconds and a max delay of 1000 seconds. + RateLimiter workqueue.TypedRateLimiter[T] + MetricProvider workqueue.MetricsProvider + Log logr.Logger +} + +// Opt allows to configure a PriorityQueue. +type Opt[T comparable] func(*Opts[T]) + +// New constructs a new PriorityQueue. +func New[T comparable](name string, o ...Opt[T]) PriorityQueue[T] { + opts := &Opts[T]{} + for _, f := range o { + f(opts) + } + + if opts.RateLimiter == nil { + opts.RateLimiter = workqueue.NewTypedItemExponentialFailureRateLimiter[T](5*time.Millisecond, 1000*time.Second) + } + + if opts.MetricProvider == nil { + opts.MetricProvider = metrics.WorkqueueMetricsProvider{} + } + + pq := &priorityqueue[T]{ + log: opts.Log, + items: map[T]*item[T]{}, + queue: btree.NewG(32, less[T]), + becameReady: sets.Set[T]{}, + metrics: newQueueMetrics[T](opts.MetricProvider, name, clock.RealClock{}), + // itemOrWaiterAdded indicates that an item or + // waiter was added. It must be buffered, because + // if we currently process items we can't tell + // if that included the new item/waiter. + itemOrWaiterAdded: make(chan struct{}, 1), + rateLimiter: opts.RateLimiter, + locked: sets.Set[T]{}, + done: make(chan struct{}), + get: make(chan item[T]), + now: time.Now, + tick: time.Tick, + } + + go pq.spin() + go pq.logState() + if _, ok := pq.metrics.(noMetrics[T]); !ok { + go pq.updateUnfinishedWorkLoop() + } + + return pq +} + +type priorityqueue[T comparable] struct { + log logr.Logger + // lock has to be acquired for any access any of items, queue, addedCounter + // or becameReady + lock sync.Mutex + items map[T]*item[T] + queue bTree[*item[T]] + + // addedCounter is a counter of elements added, we need it + // because unixNano is not guaranteed to be unique. + addedCounter uint64 + + // becameReady holds items that are in the queue, were added + // with non-zero after and became ready. We need it to call the + // metrics add exactly once for them. + becameReady sets.Set[T] + metrics queueMetrics[T] + + itemOrWaiterAdded chan struct{} + + rateLimiter workqueue.TypedRateLimiter[T] + + // locked contains the keys we handed out through Get() and that haven't + // yet been returned through Done(). + locked sets.Set[T] + lockedLock sync.RWMutex + + shutdown atomic.Bool + done chan struct{} + + get chan item[T] + + // waiters is the number of routines blocked in Get, we use it to determine + // if we can push items. + waiters atomic.Int64 + + // Configurable for testing + now func() time.Time + tick func(time.Duration) <-chan time.Time +} + +func (w *priorityqueue[T]) AddWithOpts(o AddOpts, items ...T) { + if w.shutdown.Load() { + return + } + + w.lock.Lock() + defer w.lock.Unlock() + + for _, key := range items { + after := o.After + if o.RateLimited { + rlAfter := w.rateLimiter.When(key) + if after == 0 || rlAfter < after { + after = rlAfter + } + } + + var readyAt *time.Time + if after > 0 { + readyAt = ptr.To(w.now().Add(after)) + w.metrics.retry() + } + if _, ok := w.items[key]; !ok { + item := &item[T]{ + Key: key, + AddedCounter: w.addedCounter, + Priority: ptr.Deref(o.Priority, 0), + ReadyAt: readyAt, + } + w.items[key] = item + w.queue.ReplaceOrInsert(item) + if item.ReadyAt == nil { + w.metrics.add(key, item.Priority) + } + w.addedCounter++ + continue + } + + // The b-tree de-duplicates based on ordering and any change here + // will affect the order - Just delete and re-add. + item, _ := w.queue.Delete(w.items[key]) + if newPriority := ptr.Deref(o.Priority, 0); newPriority > item.Priority { + // Update depth metric only if the item in the queue was already added to the depth metric. + if item.ReadyAt == nil || w.becameReady.Has(key) { + w.metrics.updateDepthWithPriorityMetric(item.Priority, newPriority) + } + item.Priority = newPriority + } + + if item.ReadyAt != nil && (readyAt == nil || readyAt.Before(*item.ReadyAt)) { + if readyAt == nil && !w.becameReady.Has(key) { + w.metrics.add(key, item.Priority) + } + item.ReadyAt = readyAt + } + + w.queue.ReplaceOrInsert(item) + } + + if len(items) > 0 { + w.notifyItemOrWaiterAdded() + } +} + +func (w *priorityqueue[T]) notifyItemOrWaiterAdded() { + select { + case w.itemOrWaiterAdded <- struct{}{}: + default: + } +} + +func (w *priorityqueue[T]) spin() { + blockForever := make(chan time.Time) + var nextReady <-chan time.Time + nextReady = blockForever + var nextItemReadyAt time.Time + + for { + select { + case <-w.done: + return + case <-w.itemOrWaiterAdded: + case <-nextReady: + nextReady = blockForever + nextItemReadyAt = time.Time{} + } + + func() { + w.lock.Lock() + defer w.lock.Unlock() + + w.lockedLock.Lock() + defer w.lockedLock.Unlock() + + // manipulating the tree from within Ascend might lead to panics, so + // track what we want to delete and do it after we are done ascending. + var toDelete []*item[T] + + var key T + + // Items in the queue tree are sorted first by priority and second by readiness, so + // items with a lower priority might be ready further down in the queue. + // We iterate through the priorities high to low until we find a ready item + pivot := item[T]{ + Key: key, + AddedCounter: 0, + Priority: math.MaxInt, + ReadyAt: nil, + } + + for { + pivotChange := false + + w.queue.AscendGreaterOrEqual(&pivot, func(item *item[T]) bool { + // Item is locked, we can not hand it out + if w.locked.Has(item.Key) { + return true + } + + if item.ReadyAt != nil { + if readyAt := item.ReadyAt.Sub(w.now()); readyAt > 0 { + if nextItemReadyAt.After(*item.ReadyAt) || nextItemReadyAt.IsZero() { + nextReady = w.tick(readyAt) + nextItemReadyAt = *item.ReadyAt + } + + // Adjusting the pivot item moves the ascend to the next lower priority + pivot.Priority = item.Priority - 1 + pivotChange = true + return false + } + if !w.becameReady.Has(item.Key) { + w.metrics.add(item.Key, item.Priority) + w.becameReady.Insert(item.Key) + } + } + + if w.waiters.Load() == 0 { + // Have to keep iterating here to ensure we update metrics + // for further items that became ready and set nextReady. + return true + } + + w.metrics.get(item.Key, item.Priority) + w.locked.Insert(item.Key) + w.waiters.Add(-1) + delete(w.items, item.Key) + toDelete = append(toDelete, item) + w.becameReady.Delete(item.Key) + w.get <- *item + + return true + }) + + if !pivotChange { + break + } + } + + for _, item := range toDelete { + w.queue.Delete(item) + } + }() + } +} + +func (w *priorityqueue[T]) Add(item T) { + w.AddWithOpts(AddOpts{}, item) +} + +func (w *priorityqueue[T]) AddAfter(item T, after time.Duration) { + w.AddWithOpts(AddOpts{After: after}, item) +} + +func (w *priorityqueue[T]) AddRateLimited(item T) { + w.AddWithOpts(AddOpts{RateLimited: true}, item) +} + +func (w *priorityqueue[T]) GetWithPriority() (_ T, priority int, shutdown bool) { + if w.shutdown.Load() { + var zero T + return zero, 0, true + } + + w.waiters.Add(1) + + w.notifyItemOrWaiterAdded() + + select { + case <-w.done: + // Return if the queue was shutdown while we were already waiting for an item here. + // For example controller workers are continuously calling GetWithPriority and + // GetWithPriority is blocking the workers if there are no items in the queue. + // If the controller and accordingly the queue is then shut down, without this code + // branch the controller workers remain blocked here and are unable to shut down. + var zero T + return zero, 0, true + case item := <-w.get: + return item.Key, item.Priority, w.shutdown.Load() + } +} + +func (w *priorityqueue[T]) Get() (item T, shutdown bool) { + key, _, shutdown := w.GetWithPriority() + return key, shutdown +} + +func (w *priorityqueue[T]) Forget(item T) { + w.rateLimiter.Forget(item) +} + +func (w *priorityqueue[T]) NumRequeues(item T) int { + return w.rateLimiter.NumRequeues(item) +} + +func (w *priorityqueue[T]) ShuttingDown() bool { + return w.shutdown.Load() +} + +func (w *priorityqueue[T]) Done(item T) { + w.lockedLock.Lock() + defer w.lockedLock.Unlock() + w.locked.Delete(item) + w.metrics.done(item) + w.notifyItemOrWaiterAdded() +} + +func (w *priorityqueue[T]) ShutDown() { + w.shutdown.Store(true) + close(w.done) +} + +// ShutDownWithDrain just calls ShutDown, as the draining +// functionality is not used by controller-runtime. +func (w *priorityqueue[T]) ShutDownWithDrain() { + w.ShutDown() +} + +// Len returns the number of items that are ready to be +// picked up. It does not include items that are not yet +// ready. +func (w *priorityqueue[T]) Len() int { + w.lock.Lock() + defer w.lock.Unlock() + + var result int + w.queue.Ascend(func(item *item[T]) bool { + if item.ReadyAt == nil || item.ReadyAt.Compare(w.now()) <= 0 { + result++ + return true + } + return false + }) + + return result +} + +func (w *priorityqueue[T]) logState() { + t := time.Tick(10 * time.Second) + for { + select { + case <-w.done: + return + case <-t: + } + + // Log level may change at runtime, so keep the + // loop going even if a given level is currently + // not enabled. + if !w.log.V(5).Enabled() { + continue + } + w.lock.Lock() + items := make([]*item[T], 0, len(w.items)) + w.queue.Ascend(func(item *item[T]) bool { + items = append(items, item) + return true + }) + w.lock.Unlock() + + w.log.V(5).Info("workqueue_items", "items", items) + } +} + +func less[T comparable](a, b *item[T]) bool { + if a.Priority != b.Priority { + return a.Priority > b.Priority + } + if a.ReadyAt == nil && b.ReadyAt != nil { + return true + } + if b.ReadyAt == nil && a.ReadyAt != nil { + return false + } + if a.ReadyAt != nil && b.ReadyAt != nil && !a.ReadyAt.Equal(*b.ReadyAt) { + return a.ReadyAt.Before(*b.ReadyAt) + } + + return a.AddedCounter < b.AddedCounter +} + +type item[T comparable] struct { + Key T `json:"key"` + AddedCounter uint64 `json:"addedCounter"` + Priority int `json:"priority"` + ReadyAt *time.Time `json:"readyAt,omitempty"` +} + +func (w *priorityqueue[T]) updateUnfinishedWorkLoop() { + t := time.Tick(500 * time.Millisecond) // borrowed from workqueue: https://github.com/kubernetes/kubernetes/blob/67a807bf142c7a2a5ecfdb2a5d24b4cdea4cc79c/staging/src/k8s.io/client-go/util/workqueue/queue.go#L182 + for { + select { + case <-w.done: + return + case <-t: + } + w.metrics.updateUnfinishedWork() + } +} + +type bTree[T any] interface { + ReplaceOrInsert(item T) (_ T, _ bool) + Delete(item T) (T, bool) + Ascend(iterator btree.ItemIteratorG[T]) + AscendGreaterOrEqual(pivot T, iterator btree.ItemIteratorG[T]) +} diff --git a/pkg/controller/priorityqueue/priorityqueue_suite_test.go b/pkg/controller/priorityqueue/priorityqueue_suite_test.go new file mode 100644 index 0000000000..71bc5ba049 --- /dev/null +++ b/pkg/controller/priorityqueue/priorityqueue_suite_test.go @@ -0,0 +1,13 @@ +package priorityqueue + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestControllerWorkqueue(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ControllerWorkqueue Suite") +} diff --git a/pkg/controller/priorityqueue/priorityqueue_test.go b/pkg/controller/priorityqueue/priorityqueue_test.go new file mode 100644 index 0000000000..e18a6393eb --- /dev/null +++ b/pkg/controller/priorityqueue/priorityqueue_test.go @@ -0,0 +1,775 @@ +package priorityqueue + +import ( + "fmt" + "math/rand/v2" + "sync" + "testing" + "time" + + fuzz "github.com/google/gofuzz" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" +) + +var _ = Describe("Controllerworkqueue", func() { + It("returns an item", func() { + q, metrics := newQueue() + defer q.ShutDown() + q.AddWithOpts(AddOpts{}, "foo") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + Expect(metrics.retries["test"]).To(Equal(0)) + }) + + It("returns items in order", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("bar")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + }) + + It("doesn't return an item that is currently locked", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("bar")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + Expect(metrics.adds["test"]).To(Equal(3)) + }) + + It("returns an item as soon as its unlocked", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("bar")) + + q.AddWithOpts(AddOpts{}, "baz") + q.Done("foo") + item, _, _ = q.GetWithPriority() + Expect(item).To(Equal("foo")) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + Expect(metrics.adds["test"]).To(Equal(4)) + }) + + It("de-duplicates items", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "foo") + + Expect(q.Len()).To(Equal(1)) + + q.lockedLock.Lock() + Expect(q.locked.Len()).To(Equal(0)) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("retains the highest priority", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{Priority: ptr.To(1)}, "foo") + q.AddWithOpts(AddOpts{Priority: ptr.To(2)}, "foo") + + item, priority, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + Expect(priority).To(Equal(2)) + + Expect(q.Len()).To(Equal(0)) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{1: 0, 2: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("gets pushed to the front if the priority increases", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{}, "foo") + q.AddWithOpts(AddOpts{}, "bar") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{Priority: ptr.To(1)}, "baz") + + item, priority, _ := q.GetWithPriority() + Expect(item).To(Equal("baz")) + Expect(priority).To(Equal(1)) + + Expect(q.Len()).To(Equal(2)) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2, 1: 0})) + Expect(metrics.adds["test"]).To(Equal(3)) + }) + + It("retains the lowest after duration", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: 0}, "foo") + q.AddWithOpts(AddOpts{After: time.Hour}, "foo") + + item, priority, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + Expect(priority).To(Equal(0)) + + Expect(q.Len()).To(Equal(0)) + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("returns an item only after after has passed", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + Expect(d).To(Equal(time.Second)) + return originalTick(d) + } + + retrievedItem := make(chan struct{}) + + go func() { + defer GinkgoRecover() + q.GetWithPriority() + close(retrievedItem) + }() + + q.AddWithOpts(AddOpts{After: time.Second}, "foo") + + Consistently(retrievedItem).ShouldNot(BeClosed()) + + forwardQueueTimeBy(time.Second) + Eventually(retrievedItem).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + Expect(metrics.retries["test"]).To(Equal(1)) + }) + + It("returns high priority item that became ready before low priority item", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + tickSetup := make(chan any) + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + Expect(d).To(Equal(time.Second)) + close(tickSetup) + return originalTick(d) + } + + lowPriority := -100 + highPriority := 0 + q.AddWithOpts(AddOpts{After: 0, Priority: &lowPriority}, "foo") + q.AddWithOpts(AddOpts{After: time.Second, Priority: &highPriority}, "prio") + + Eventually(tickSetup).Should(BeClosed()) + + forwardQueueTimeBy(1 * time.Second) + key, prio, _ := q.GetWithPriority() + + Expect(key).To(Equal("prio")) + Expect(prio).To(Equal(0)) + Expect(metrics.depth["test"]).To(Equal(map[int]int{-100: 1, 0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + Expect(metrics.retries["test"]).To(Equal(1)) + }) + + It("returns an item to a waiter as soon as it has one", func() { + q, metrics := newQueue() + defer q.ShutDown() + + retrieved := make(chan struct{}) + go func() { + defer GinkgoRecover() + item, _, _ := q.GetWithPriority() + Expect(item).To(Equal("foo")) + close(retrieved) + }() + + // We are waiting for the GetWithPriority() call to be blocked + // on retrieving an item. As golang doesn't provide a way to + // check if something is listening on a channel without + // sending them a message, I can't think of a way to do this + // without sleeping. + time.Sleep(time.Second) + q.AddWithOpts(AddOpts{}, "foo") + Eventually(retrieved).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(1)) + }) + + It("returns multiple items with after in correct order", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + // What a bunch of bs. Deferring in here causes + // ginkgo to deadlock, presumably because it + // never returns after the defer. Not deferring + // hides the actual assertion result and makes + // it complain that there should be a defer. + // Move the assertion into a goroutine just to + // get around that mess. + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + + // This is not deterministic and depends on which of + // Add() or Spin() gets the lock first. + Expect(d).To(Or(Equal(200*time.Millisecond), Equal(time.Second))) + }() + <-done + return originalTick(d) + } + + retrievedItem := make(chan struct{}) + retrievedSecondItem := make(chan struct{}) + + go func() { + defer GinkgoRecover() + first, _, _ := q.GetWithPriority() + Expect(first).To(Equal("bar")) + close(retrievedItem) + + second, _, _ := q.GetWithPriority() + Expect(second).To(Equal("foo")) + close(retrievedSecondItem) + }() + + q.AddWithOpts(AddOpts{After: time.Second}, "foo") + q.AddWithOpts(AddOpts{After: 200 * time.Millisecond}, "bar") + + Consistently(retrievedItem).ShouldNot(BeClosed()) + + forwardQueueTimeBy(time.Second) + Eventually(retrievedItem).Should(BeClosed()) + Eventually(retrievedSecondItem).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + }) + + It("doesn't include non-ready items in Len()", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: time.Minute}, "foo") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{After: time.Minute}, "bar") + q.AddWithOpts(AddOpts{}, "bal") + + Expect(q.Len()).To(Equal(2)) + Expect(metrics.depth).To(HaveLen(1)) + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2})) + }) + + // ref: https://github.com/kubernetes-sigs/controller-runtime/issues/3239 + It("Get from priority queue might get stuck when the priority queue is shut down", func() { + q, _ := newQueue() + + q.Add("baz") + // shut down + q.ShutDown() + q.AddWithOpts(AddOpts{After: time.Second}, "foo") + + item, priority, isShutDown := q.GetWithPriority() + Expect(item).To(Equal("")) + Expect(priority).To(Equal(0)) + Expect(isShutDown).To(BeTrue()) + + item1, priority1, isShutDown := q.GetWithPriority() + Expect(item1).To(Equal("")) + Expect(priority1).To(Equal(0)) + Expect(isShutDown).To(BeTrue()) + }) + + It("Get from priority queue should get unblocked when the priority queue is shut down", func() { + q, _ := newQueue() + + getUnblocked := make(chan struct{}) + + go func() { + defer GinkgoRecover() + defer close(getUnblocked) + + item, priority, isShutDown := q.GetWithPriority() + Expect(item).To(Equal("")) + Expect(priority).To(Equal(0)) + Expect(isShutDown).To(BeTrue()) + }() + + // Verify the go routine above is now waiting for an item. + Eventually(q.waiters.Load).Should(Equal(int64(1))) + Consistently(getUnblocked).ShouldNot(BeClosed()) + + // shut down + q.ShutDown() + + // Verify the shutdown unblocked the go routine. + Eventually(getUnblocked).Should(BeClosed()) + }) + + It("items are included in Len() and the queueDepth metric once they are ready", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "foo") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "bar") + q.AddWithOpts(AddOpts{}, "bal") + + Expect(q.Len()).To(Equal(2)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2})) + metrics.mu.Unlock() + time.Sleep(time.Second) + Expect(q.Len()).To(Equal(4)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 4})) + metrics.mu.Unlock() + + // Drain queue + for range 4 { + item, _ := q.Get() + q.Done(item) + } + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + metrics.mu.Unlock() + + // Validate that doing it again still works to notice bugs with removing + // it from the queues becameReady tracking. + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "foo") + q.AddWithOpts(AddOpts{}, "baz") + q.AddWithOpts(AddOpts{After: 500 * time.Millisecond}, "bar") + q.AddWithOpts(AddOpts{}, "bal") + + Expect(q.Len()).To(Equal(2)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 2})) + metrics.mu.Unlock() + time.Sleep(time.Second) + Expect(q.Len()).To(Equal(4)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 4})) + metrics.mu.Unlock() + }) + + It("returns many items", func() { + // This test ensures the queue is able to drain a large queue without panic'ing. + // In a previous version of the code we were calling queue.Delete within q.Ascend + // which led to a panic in queue.Ascend > iterate: + // "panic: runtime error: index out of range [0] with length 0" + q, _ := newQueue() + defer q.ShutDown() + + for range 20 { + for i := range 1000 { + rn := rand.N(100) + if rn < 10 { + q.AddWithOpts(AddOpts{After: time.Duration(rn) * time.Millisecond}, fmt.Sprintf("foo%d", i)) + } else { + q.AddWithOpts(AddOpts{Priority: &rn}, fmt.Sprintf("foo%d", i)) + } + } + + wg := sync.WaitGroup{} + for range 100 { // The panic only occurred relatively frequently with a high number of go routines. + wg.Add(1) + go func() { + defer wg.Done() + for range 10 { + obj, _, _ := q.GetWithPriority() + q.Done(obj) + } + }() + } + + wg.Wait() + } + }) + + It("updates metrics correctly for an item that gets initially added with after and then without", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: time.Hour}, "foo") + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{})) + metrics.mu.Unlock() + + q.AddWithOpts(AddOpts{}, "foo") + + Expect(q.Len()).To(Equal(1)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + metrics.mu.Unlock() + + // Get the item to ensure the codepath in + // `spin` for the metrics is passed by so + // that this starts failing if it incorrectly + // calls `metrics.add` again. + item, _ := q.Get() + Expect(item).To(Equal("foo")) + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + metrics.mu.Unlock() + }) + + It("Updates metrics correctly for an item whose requeueAfter expired that gets added again without requeueAfter", func() { + q, metrics := newQueue() + defer q.ShutDown() + + q.AddWithOpts(AddOpts{After: 50 * time.Millisecond}, "foo") + time.Sleep(100 * time.Millisecond) + + Expect(q.Len()).To(Equal(1)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + metrics.mu.Unlock() + + q.AddWithOpts(AddOpts{}, "foo") + Expect(q.Len()).To(Equal(1)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 1})) + metrics.mu.Unlock() + + // Get the item to ensure the codepath in + // `spin` for the metrics is passed by so + // that this starts failing if it incorrectly + // calls `metrics.add` again. + item, _ := q.Get() + Expect(item).To(Equal("foo")) + Expect(q.Len()).To(Equal(0)) + metrics.mu.Lock() + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + metrics.mu.Unlock() + }) + + It("When adding items with rateLimit, previous items' rateLimit should not affect subsequent items", func() { + q, metrics, forwardQueueTimeBy := newQueueWithTimeForwarder() + defer q.ShutDown() + + q.rateLimiter = workqueue.NewTypedItemExponentialFailureRateLimiter[string](5*time.Millisecond, 1000*time.Second) + originalTick := q.tick + q.tick = func(d time.Duration) <-chan time.Time { + done := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(done) + + Expect(d).To(Or(Equal(5*time.Millisecond), Equal(635*time.Millisecond))) + }() + <-done + return originalTick(d) + } + + retrievedItem := make(chan struct{}) + retrievedSecondItem := make(chan struct{}) + + go func() { + defer GinkgoRecover() + first, _, _ := q.GetWithPriority() + Expect(first).To(Equal("foo")) + close(retrievedItem) + + second, _, _ := q.GetWithPriority() + Expect(second).To(Equal("bar")) + close(retrievedSecondItem) + }() + + // after 7 calls, the next When("bar") call will return 640ms. + for range 7 { + q.rateLimiter.When("bar") + } + q.AddWithOpts(AddOpts{RateLimited: true}, "foo", "bar") + + Consistently(retrievedItem).ShouldNot(BeClosed()) + forwardQueueTimeBy(5 * time.Millisecond) + Eventually(retrievedItem).Should(BeClosed()) + + Consistently(retrievedSecondItem).ShouldNot(BeClosed()) + forwardQueueTimeBy(635 * time.Millisecond) + Eventually(retrievedSecondItem).Should(BeClosed()) + + Expect(metrics.depth["test"]).To(Equal(map[int]int{0: 0})) + Expect(metrics.adds["test"]).To(Equal(2)) + Expect(metrics.retries["test"]).To(Equal(2)) + }) +}) + +func BenchmarkAddGetDone(b *testing.B) { + q := New[int]("") + defer q.ShutDown() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < 1000; i++ { + q.Add(i) + } + for range 1000 { + item, _ := q.Get() + q.Done(item) + } + } +} + +func BenchmarkAddOnly(b *testing.B) { + q := New[int]("") + defer q.ShutDown() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < 1000; i++ { + q.Add(i) + } + } +} + +func BenchmarkAddLockContended(b *testing.B) { + q := New[int]("") + defer q.ShutDown() + go func() { + for range 1000 { + item, _ := q.Get() + q.Done(item) + } + }() + b.ResetTimer() + for n := 0; n < b.N; n++ { + for i := 0; i < 1000; i++ { + q.Add(i) + } + } +} + +// TestFuzzPrioriorityQueue validates a set of basic +// invariants that should always be true: +// +// - The queue is threadsafe when multiple producers and consumers +// are involved +// - There are no deadlocks +// - An item is never handed out again before it is returned +// - Items in the queue are de-duplicated +// - max(existing priority, new priority) is used +func TestFuzzPriorityQueue(t *testing.T) { + t.Parallel() + + seed := time.Now().UnixNano() + t.Logf("seed: %d", seed) + f := fuzz.NewWithSeed(seed) + fuzzLock := sync.Mutex{} + fuzz := func(in any) { + fuzzLock.Lock() + defer fuzzLock.Unlock() + + f.Fuzz(in) + } + + inQueue := map[string]int{} + inQueueLock := sync.Mutex{} + + handedOut := sets.Set[string]{} + handedOutLock := sync.Mutex{} + + wg := sync.WaitGroup{} + q, metrics := newQueue() + + for range 10 { + wg.Add(1) + go func() { + defer wg.Done() + + for range 1000 { + opts, item := AddOpts{}, "" + + fuzz(&opts) + fuzz(&item) + + if opts.After > 100*time.Millisecond { + opts.After = 10 * time.Millisecond + } + opts.RateLimited = false + + func() { + inQueueLock.Lock() + defer inQueueLock.Unlock() + + q.AddWithOpts(opts, item) + if existingPriority, exists := inQueue[item]; !exists || existingPriority < ptr.Deref(opts.Priority, 0) { + inQueue[item] = ptr.Deref(opts.Priority, 0) + } + }() + } + }() + } + + for range 100 { + wg.Add(1) + + go func() { + defer wg.Done() + + for { + item, cont := func() (string, bool) { + inQueueLock.Lock() + defer inQueueLock.Unlock() + + if len(inQueue) == 0 { + return "", false + } + + item, priority, _ := q.GetWithPriority() + if expected := inQueue[item]; expected != priority { + t.Errorf("got priority %d, expected %d", priority, expected) + } + delete(inQueue, item) + return item, true + }() + + if !cont { + return + } + + func() { + handedOutLock.Lock() + defer handedOutLock.Unlock() + + if handedOut.Has(item) { + t.Errorf("item %s got handed out more than once", item) + } + + metrics.mu.Lock() + for priority, depth := range metrics.depth["test"] { + if depth < 0 { + t.Errorf("negative depth of %d for priority %d:", depth, priority) + } + } + + metrics.mu.Unlock() + handedOut.Insert(item) + }() + + func() { + handedOutLock.Lock() + defer handedOutLock.Unlock() + + handedOut.Delete(item) + q.Done(item) + }() + } + }() + } + + wg.Wait() +} + +func newQueueWithTimeForwarder() (_ *priorityqueue[string], _ *fakeMetricsProvider, forwardQueueTime func(time.Duration)) { + q, m := newQueue() + + now := time.Now().Round(time.Second) + nowLock := sync.Mutex{} + tick := make(chan time.Time) + + q.now = func() time.Time { + nowLock.Lock() + defer nowLock.Unlock() + return now + } + q.tick = func(d time.Duration) <-chan time.Time { + return tick + } + + return q, m, func(d time.Duration) { + nowLock.Lock() + now = now.Add(d) + nowLock.Unlock() + tick <- now + } +} + +func newQueue() (*priorityqueue[string], *fakeMetricsProvider) { + metrics := newFakeMetricsProvider() + q := New("test", func(o *Opts[string]) { + o.MetricProvider = metrics + }) + q.(*priorityqueue[string]).queue = &btreeInteractionValidator{ + bTree: q.(*priorityqueue[string]).queue, + } + + // validate that tick always gets a positive value as it will just return + // nil otherwise, which results in blocking forever. + upstreamTick := q.(*priorityqueue[string]).tick + q.(*priorityqueue[string]).tick = func(d time.Duration) <-chan time.Time { + if d <= 0 { + panic(fmt.Sprintf("got non-positive tick: %v", d)) + } + return upstreamTick(d) + } + return q.(*priorityqueue[string]), metrics +} + +type btreeInteractionValidator struct { + bTree[*item[string]] +} + +func (b *btreeInteractionValidator) ReplaceOrInsert(item *item[string]) (*item[string], bool) { + // There is no codepath that updates an item + item, alreadyExist := b.bTree.ReplaceOrInsert(item) + if alreadyExist { + panic(fmt.Sprintf("ReplaceOrInsert: item %v already existed", item)) + } + return item, alreadyExist +} + +func (b *btreeInteractionValidator) Delete(item *item[string]) (*item[string], bool) { + // There is no codepath that deletes an item that doesn't exist + old, existed := b.bTree.Delete(item) + if !existed { + panic(fmt.Sprintf("Delete: item %v not found", item)) + } + return old, existed +} diff --git a/pkg/doc.go b/pkg/doc.go index 89b380c108..64693b4829 100644 --- a/pkg/doc.go +++ b/pkg/doc.go @@ -137,11 +137,11 @@ Source provides event: EventHandler enqueues Request: -* &handler.EnqueueRequestForObject{} -> (reconcile.Request{types.NamespaceName{Name: "foo", Namespace: "bar"}}) +* &handler.EnqueueRequestForObject{} -> (reconcile.Request{types.NamespaceName{Namespace: "foo", Name: "bar"}}) Reconciler is called with the Request: -* Reconciler(reconcile.Request{types.NamespaceName{Name: "foo", Namespace: "bar"}}) +* Reconciler(reconcile.Request{types.NamespaceName{Namespace: "foo", Name: "bar"}}) # Usage diff --git a/pkg/envtest/binaries.go b/pkg/envtest/binaries.go new file mode 100644 index 0000000000..5110d32658 --- /dev/null +++ b/pkg/envtest/binaries.go @@ -0,0 +1,387 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "k8s.io/apimachinery/pkg/util/version" + "sigs.k8s.io/yaml" +) + +// DefaultBinaryAssetsIndexURL is the default index used in HTTPClient. +var DefaultBinaryAssetsIndexURL = "https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml" + +// SetupEnvtestDefaultBinaryAssetsDirectory returns the default location that setup-envtest uses to store envtest binaries. +// Setting BinaryAssetsDirectory to this directory allows sharing envtest binaries with setup-envtest. +// +// The directory is dependent on operating system: +// +// - Windows: %LocalAppData%\kubebuilder-envtest +// - OSX: ~/Library/Application Support/io.kubebuilder.envtest +// - Others: ${XDG_DATA_HOME:-~/.local/share}/kubebuilder-envtest +// +// Otherwise, it errors out. Note that these paths must not be relied upon +// manually. +func SetupEnvtestDefaultBinaryAssetsDirectory() (string, error) { + var baseDir string + + // find the base data directory + switch runtime.GOOS { + case "windows": + baseDir = os.Getenv("LocalAppData") + if baseDir == "" { + return "", errors.New("%LocalAppData% is not defined") + } + case "darwin": + homeDir := os.Getenv("HOME") + if homeDir == "" { + return "", errors.New("$HOME is not defined") + } + baseDir = filepath.Join(homeDir, "Library/Application Support") + default: + baseDir = os.Getenv("XDG_DATA_HOME") + if baseDir == "" { + homeDir := os.Getenv("HOME") + if homeDir == "" { + return "", errors.New("neither $XDG_DATA_HOME nor $HOME are defined") + } + baseDir = filepath.Join(homeDir, ".local/share") + } + } + + // append our program-specific dir to it (OSX has a slightly different + // convention so try to follow that). + switch runtime.GOOS { + case "darwin", "ios": + return filepath.Join(baseDir, "io.kubebuilder.envtest", "k8s"), nil + default: + return filepath.Join(baseDir, "kubebuilder-envtest", "k8s"), nil + } +} + +// index represents an index of envtest binary archives. Example: +// +// releases: +// v1.28.0: +// envtest-v1.28.0-darwin-amd64.tar.gz: +// hash: +// selfLink: +type index struct { + // Releases maps Kubernetes versions to Releases (envtest archives). + Releases map[string]release `json:"releases"` +} + +// release maps an archive name to an archive. +type release map[string]archive + +// archive contains the self link to an archive and its hash. +type archive struct { + Hash string `json:"hash"` + SelfLink string `json:"selfLink"` +} + +// parseKubernetesVersion returns: +// 1. the SemVer form of s when it refers to a specific Kubernetes release, or +// 2. the major and minor portions of s when it refers to a release series, or +// 3. an error +func parseKubernetesVersion(s string) (exact string, major, minor uint, err error) { + if v, err := version.ParseSemantic(s); err == nil { + return v.String(), 0, 0, nil + } + + // See two parseable components and nothing else. + if v, err := version.ParseGeneric(s); err == nil && len(v.Components()) == 2 { + if v.String() == strings.TrimPrefix(s, "v") { + return "", v.Major(), v.Minor(), nil + } + } + + return "", 0, 0, fmt.Errorf("could not parse %q as version", s) +} + +func downloadBinaryAssets(ctx context.Context, binaryAssetsDirectory, binaryAssetsVersion, binaryAssetsIndexURL string) (string, string, string, error) { + if binaryAssetsIndexURL == "" { + binaryAssetsIndexURL = DefaultBinaryAssetsIndexURL + } + + downloadRootDir := binaryAssetsDirectory + if downloadRootDir == "" { + var err error + if downloadRootDir, err = os.MkdirTemp("", "envtest-binaries-"); err != nil { + return "", "", "", fmt.Errorf("failed to create tmp directory for envtest binaries: %w", err) + } + } + + var binaryAssetsIndex *index + switch exact, major, minor, err := parseKubernetesVersion(binaryAssetsVersion); { + case binaryAssetsVersion != "" && err != nil: + return "", "", "", err + + case binaryAssetsVersion != "" && exact != "": + // Look for these specific binaries locally before downloading them from the release index. + // Use the canonical form of the version from here on. + binaryAssetsVersion = "v" + exact + + case binaryAssetsVersion == "" || major != 0 || minor != 0: + // Select a stable version from the release index before continuing. + binaryAssetsIndex, err = getIndex(ctx, binaryAssetsIndexURL) + if err != nil { + return "", "", "", err + } + + binaryAssetsVersion, err = latestStableVersionFromIndex(binaryAssetsIndex, major, minor) + if err != nil { + return "", "", "", err + } + } + + // Storing the envtest binaries in a directory structure that is compatible with setup-envtest. + // This makes it possible to share the envtest binaries with setup-envtest if the BinaryAssetsDirectory is set to SetupEnvtestDefaultBinaryAssetsDirectory(). + downloadDir := path.Join(downloadRootDir, fmt.Sprintf("%s-%s-%s", strings.TrimPrefix(binaryAssetsVersion, "v"), runtime.GOOS, runtime.GOARCH)) + if !fileExists(downloadDir) { + if err := os.MkdirAll(downloadDir, 0700); err != nil { + return "", "", "", fmt.Errorf("failed to create directory %q for envtest binaries: %w", downloadDir, err) + } + } + + apiServerPath := path.Join(downloadDir, "kube-apiserver") + etcdPath := path.Join(downloadDir, "etcd") + kubectlPath := path.Join(downloadDir, "kubectl") + + if fileExists(apiServerPath) && fileExists(etcdPath) && fileExists(kubectlPath) { + // Nothing to do if the binaries already exist. + return apiServerPath, etcdPath, kubectlPath, nil + } + + // Get Index if we didn't have to get it above to get the latest stable version. + if binaryAssetsIndex == nil { + var err error + binaryAssetsIndex, err = getIndex(ctx, binaryAssetsIndexURL) + if err != nil { + return "", "", "", err + } + } + + buf := &bytes.Buffer{} + if err := downloadBinaryAssetsArchive(ctx, binaryAssetsIndex, binaryAssetsVersion, buf); err != nil { + return "", "", "", err + } + + gzStream, err := gzip.NewReader(buf) + if err != nil { + return "", "", "", fmt.Errorf("failed to create gzip reader to extract envtest binaries: %w", err) + } + tarReader := tar.NewReader(gzStream) + + var header *tar.Header + for header, err = tarReader.Next(); err == nil; header, err = tarReader.Next() { + if header.Typeflag != tar.TypeReg { + // Skip non-regular file entry in archive. + continue + } + + // Just dump all files directly into the download directory, ignoring the prefixed directory paths. + // We also ignore bits for the most part (except for X). + fileName := filepath.Base(header.Name) + perms := 0555 & header.Mode // make sure we're at most r+x + + // Setting O_EXCL to get an error if the file already exists. + f, err := os.OpenFile(path.Join(downloadDir, fileName), os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_TRUNC, os.FileMode(perms)) + if err != nil { + if os.IsExist(err) { + // Nothing to do if the file already exists. We assume another process created the file concurrently. + continue + } + return "", "", "", fmt.Errorf("failed to create file %s in directory %s: %w", fileName, downloadDir, err) + } + if err := func() error { + defer f.Close() + if _, err := io.Copy(f, tarReader); err != nil { + return fmt.Errorf("failed to write file %s in directory %s: %w", fileName, downloadDir, err) + } + return nil + }(); err != nil { + return "", "", "", fmt.Errorf("failed to close file %s in directory %s: %w", fileName, downloadDir, err) + } + } + + return apiServerPath, etcdPath, kubectlPath, nil +} + +func fileExists(path string) bool { + if _, err := os.Stat(path); err == nil { + return true + } + return false +} + +func downloadBinaryAssetsArchive(ctx context.Context, index *index, version string, out io.Writer) error { + archives, ok := index.Releases[version] + if !ok { + return fmt.Errorf("failed to find envtest binaries for version %s", version) + } + + archiveName := fmt.Sprintf("envtest-%s-%s-%s.tar.gz", version, runtime.GOOS, runtime.GOARCH) + archive, ok := archives[archiveName] + if !ok { + return fmt.Errorf("failed to find envtest binaries for version %s with archiveName %s", version, archiveName) + } + + archiveURL, err := url.Parse(archive.SelfLink) + if err != nil { + return fmt.Errorf("failed to parse envtest binaries archive URL %q: %w", archiveURL, err) + } + + req, err := http.NewRequestWithContext(ctx, "GET", archiveURL.String(), nil) + if err != nil { + return fmt.Errorf("failed to create request to download %s: %w", archiveURL.String(), err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download %s: %w", archiveURL.String(), err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("failed to download %s, got status %q", archiveURL.String(), resp.Status) + } + + return readBody(resp, out, archiveName, archive.Hash) +} + +// latestStableVersionFromIndex returns the version with highest [precedence] in index that is not a prerelease. +// When either major or minor are not zero, the returned version will have those major and minor versions. +// Note that the version cannot be limited to 0.0.x this way. +// +// It is an error when there is no appropriate version in index. +// +// [precedence]: https://semver.org/spec/v2.0.0.html#spec-item-11 +func latestStableVersionFromIndex(index *index, major, minor uint) (string, error) { + if len(index.Releases) == 0 { + return "", fmt.Errorf("failed to find latest stable version from index: index is empty") + } + + var found *version.Version + for releaseVersion := range index.Releases { + v, err := version.ParseSemantic(releaseVersion) + if err != nil { + return "", fmt.Errorf("failed to parse version %q: %w", releaseVersion, err) + } + + // Filter out pre-releases. + if len(v.PreRelease()) > 0 { + continue + } + + // Filter on release series, if any. + if (major != 0 || minor != 0) && (v.Major() != major || v.Minor() != minor) { + continue + } + + if found == nil || v.GreaterThan(found) { + found = v + } + } + + if found == nil { + search := "any" + if major != 0 || minor != 0 { + search = fmt.Sprint(major, ".", minor) + } + + return "", fmt.Errorf("failed to find latest stable version from index: index does not have %s stable versions", search) + } + + return "v" + found.String(), nil +} + +func getIndex(ctx context.Context, indexURL string) (*index, error) { + loc, err := url.Parse(indexURL) + if err != nil { + return nil, fmt.Errorf("unable to parse index URL: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return nil, fmt.Errorf("unable to construct request to get index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("unable to perform request to get index: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unable to get index -- got status %q", resp.Status) + } + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to get index -- unable to read body %w", err) + } + + var index index + if err := yaml.Unmarshal(responseBody, &index); err != nil { + return nil, fmt.Errorf("unable to unmarshal index: %w", err) + } + return &index, nil +} + +func readBody(resp *http.Response, out io.Writer, archiveName string, expectedHash string) error { + // Stream in chunks to do the checksum + buf := make([]byte, 32*1024) // 32KiB, same as io.Copy + hasher := sha512.New() + + for cont := true; cont; { + amt, err := resp.Body.Read(buf) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("unable read next chunk of %s: %w", archiveName, err) + } + if amt > 0 { + // checksum never returns errors according to docs + hasher.Write(buf[:amt]) + if _, err := out.Write(buf[:amt]); err != nil { + return fmt.Errorf("unable write next chunk of %s: %w", archiveName, err) + } + } + cont = amt > 0 && !errors.Is(err, io.EOF) + } + + actualHash := hex.EncodeToString(hasher.Sum(nil)) + if actualHash != expectedHash { + return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (expected)", archiveName, actualHash, expectedHash) + } + + return nil +} diff --git a/pkg/envtest/binaries_test.go b/pkg/envtest/binaries_test.go new file mode 100644 index 0000000000..aa83963381 --- /dev/null +++ b/pkg/envtest/binaries_test.go @@ -0,0 +1,357 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/rand" + "crypto/sha512" + "encoding/hex" + "fmt" + "net/http" + "os" + "path" + "runtime" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/ghttp" + "sigs.k8s.io/yaml" +) + +func TestParseKubernetesVersion(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + inputs []string + + expectError string + expectExact bool + expectSeriesMajor uint + expectSeriesMinor uint + }{ + { + name: `SemVer and "v" prefix are exact`, + inputs: []string{ + "1.2.3", "v1.2.3", "v1.30.2", "v1.31.0-beta.0", "v1.33.0-alpha.2", + }, + expectExact: true, + }, + { + name: "empty string is not a version", + inputs: []string{""}, + expectError: "could not parse", + }, + { + name: "leading zeroes are not a version", + inputs: []string{ + "01.2.0", "00001.2.3", "1.2.03", "v01.02.0003", + }, + expectError: "could not parse", + }, + { + name: "weird stuff is not a version", + inputs: []string{ + "asdf", "version", "vegeta4", "the.1", "2ne1", "=7.8.9", "10.x", "*", + "0.0001", "1.00002", "v1.2anything", "1.2.x", "1.2.z", "1.2.*", + }, + expectError: "could not parse", + }, + { + name: "one number is not a version", + inputs: []string{ + "1", "v1", "v001", "1.", "v1.", "1.x", + }, + expectError: "could not parse", + }, + { + name: "two numbers are a release series", + inputs: []string{"0.1", "v0.1"}, + + expectSeriesMajor: 0, + expectSeriesMinor: 1, + }, + { + name: "two numbers are a release series", + inputs: []string{"1.2", "v1.2"}, + + expectSeriesMajor: 1, + expectSeriesMinor: 2, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for _, input := range tc.inputs { + exact, major, minor, err := parseKubernetesVersion(input) + + if tc.expectError != "" && err == nil { + t.Errorf("expected error %q, got none", tc.expectError) + } + if tc.expectError != "" && !strings.Contains(err.Error(), tc.expectError) { + t.Errorf("expected error %q, got %q", tc.expectError, err) + } + if tc.expectError == "" && err != nil { + t.Errorf("expected no error, got %q", err) + continue + } + + if tc.expectExact { + if expected := strings.TrimPrefix(input, "v"); exact != expected { + t.Errorf("expected canonical %q for %q, got %q", expected, input, exact) + } + if major != 0 || minor != 0 { + t.Errorf("expected no release series for %q, got (%v, %v)", input, major, minor) + } + continue + } + + if major != tc.expectSeriesMajor { + t.Errorf("expected major %v for %q, got %v", tc.expectSeriesMajor, input, major) + } + if minor != tc.expectSeriesMinor { + t.Errorf("expected minor %v for %q, got %v", tc.expectSeriesMinor, input, minor) + } + if exact != "" { + t.Errorf("expected no canonical version for %q, got %q", input, exact) + } + } + }) + } +} + +var _ = Describe("Test download binaries", func() { + var downloadDirectory string + var server *ghttp.Server + + BeforeEach(func() { + downloadDirectory = GinkgoT().TempDir() + + server = ghttp.NewServer() + DeferCleanup(func() { + server.Close() + }) + setupServer(server) + }) + + It("should download binaries of latest stable version", func(ctx SpecContext) { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(ctx, downloadDirectory, "", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Verify latest stable version (v1.32.0) was downloaded + versionDownloadDirectory := path.Join(downloadDirectory, fmt.Sprintf("1.32.0-%s-%s", runtime.GOOS, runtime.GOARCH)) + Expect(apiServerPath).To(Equal(path.Join(versionDownloadDirectory, "kube-apiserver"))) + Expect(etcdPath).To(Equal(path.Join(versionDownloadDirectory, "etcd"))) + Expect(kubectlPath).To(Equal(path.Join(versionDownloadDirectory, "kubectl"))) + + dirEntries, err := os.ReadDir(versionDownloadDirectory) + Expect(err).ToNot(HaveOccurred()) + var actualFiles []string + for _, e := range dirEntries { + actualFiles = append(actualFiles, e.Name()) + } + Expect(actualFiles).To(ConsistOf("some-file")) + }) + + It("should download binaries of an exact version", func(ctx SpecContext) { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(ctx, downloadDirectory, "v1.31.0", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Verify exact version (v1.31.0) was downloaded + versionDownloadDirectory := path.Join(downloadDirectory, fmt.Sprintf("1.31.0-%s-%s", runtime.GOOS, runtime.GOARCH)) + Expect(apiServerPath).To(Equal(path.Join(versionDownloadDirectory, "kube-apiserver"))) + Expect(etcdPath).To(Equal(path.Join(versionDownloadDirectory, "etcd"))) + Expect(kubectlPath).To(Equal(path.Join(versionDownloadDirectory, "kubectl"))) + + dirEntries, err := os.ReadDir(versionDownloadDirectory) + Expect(err).ToNot(HaveOccurred()) + var actualFiles []string + for _, e := range dirEntries { + actualFiles = append(actualFiles, e.Name()) + } + Expect(actualFiles).To(ConsistOf("some-file")) + }) + + It("should download binaries of latest stable version of a release series", func(ctx SpecContext) { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(ctx, downloadDirectory, "1.31", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).ToNot(HaveOccurred()) + + // Verify stable version (v1.31.4) was downloaded + versionDownloadDirectory := path.Join(downloadDirectory, fmt.Sprintf("1.31.4-%s-%s", runtime.GOOS, runtime.GOARCH)) + Expect(apiServerPath).To(Equal(path.Join(versionDownloadDirectory, "kube-apiserver"))) + Expect(etcdPath).To(Equal(path.Join(versionDownloadDirectory, "etcd"))) + Expect(kubectlPath).To(Equal(path.Join(versionDownloadDirectory, "kubectl"))) + + dirEntries, err := os.ReadDir(versionDownloadDirectory) + Expect(err).ToNot(HaveOccurred()) + var actualFiles []string + for _, e := range dirEntries { + actualFiles = append(actualFiles, e.Name()) + } + Expect(actualFiles).To(ConsistOf("some-file")) + }) + + It("should error when the asset version is not a version", func(ctx SpecContext) { + _, _, _, err := downloadBinaryAssets(ctx, downloadDirectory, "wonky", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).To(MatchError(`could not parse "wonky" as version`)) + }) + + It("should error when the asset version is not in the index", func(ctx SpecContext) { + _, _, _, err := downloadBinaryAssets(ctx, downloadDirectory, "v1.5.0", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).To(MatchError("failed to find envtest binaries for version v1.5.0")) + + _, _, _, err = downloadBinaryAssets(ctx, downloadDirectory, "v1.5", fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml")) + Expect(err).To(MatchError("failed to find latest stable version from index: index does not have 1.5 stable versions")) + }) +}) + +var ( + envtestBinaryArchives = index{ + Releases: map[string]release{ + "v1.32.0": map[string]archive{ + "envtest-v1.32.0-darwin-amd64.tar.gz": {}, + "envtest-v1.32.0-darwin-arm64.tar.gz": {}, + "envtest-v1.32.0-linux-amd64.tar.gz": {}, + "envtest-v1.32.0-linux-arm64.tar.gz": {}, + "envtest-v1.32.0-linux-ppc64le.tar.gz": {}, + "envtest-v1.32.0-linux-s390x.tar.gz": {}, + "envtest-v1.32.0-windows-amd64.tar.gz": {}, + }, + "v1.31.4": map[string]archive{ + "envtest-v1.31.4-darwin-amd64.tar.gz": {}, + "envtest-v1.31.4-darwin-arm64.tar.gz": {}, + "envtest-v1.31.4-linux-amd64.tar.gz": {}, + "envtest-v1.31.4-linux-arm64.tar.gz": {}, + "envtest-v1.31.4-linux-ppc64le.tar.gz": {}, + "envtest-v1.31.4-linux-s390x.tar.gz": {}, + "envtest-v1.31.4-windows-amd64.tar.gz": {}, + }, + "v1.31.0": map[string]archive{ + "envtest-v1.31.0-darwin-amd64.tar.gz": {}, + "envtest-v1.31.0-darwin-arm64.tar.gz": {}, + "envtest-v1.31.0-linux-amd64.tar.gz": {}, + "envtest-v1.31.0-linux-arm64.tar.gz": {}, + "envtest-v1.31.0-linux-ppc64le.tar.gz": {}, + "envtest-v1.31.0-linux-s390x.tar.gz": {}, + "envtest-v1.31.0-windows-amd64.tar.gz": {}, + }, + }, + } +) + +func setupServer(server *ghttp.Server) { + itemsHTTP := makeArchives(envtestBinaryArchives) + + // The index from itemsHTTP contains only relative SelfLinks. + // finalIndex will contain the full links based on server.Addr(). + finalIndex := index{ + Releases: map[string]release{}, + } + + for releaseVersion, releases := range itemsHTTP.index.Releases { + finalIndex.Releases[releaseVersion] = release{} + + for archiveName, a := range releases { + finalIndex.Releases[releaseVersion][archiveName] = archive{ + Hash: a.Hash, + SelfLink: fmt.Sprintf("http://%s/%s", server.Addr(), a.SelfLink), + } + content := itemsHTTP.contents[archiveName] + + // Note: Using the relative path from archive here instead of the full path. + server.RouteToHandler("GET", "/"+a.SelfLink, func(resp http.ResponseWriter, req *http.Request) { + resp.WriteHeader(http.StatusOK) + Expect(resp.Write(content)).To(Equal(len(content))) + }) + } + } + + indexYAML, err := yaml.Marshal(finalIndex) + Expect(err).ToNot(HaveOccurred()) + + server.RouteToHandler("GET", "/envtest-releases.yaml", ghttp.RespondWith( + http.StatusOK, + indexYAML, + )) +} + +type itemsHTTP struct { + index index + contents map[string][]byte +} + +func makeArchives(i index) itemsHTTP { + // This creates a new copy of the index so modifying the index + // in some tests doesn't affect others. + res := itemsHTTP{ + index: index{ + Releases: map[string]release{}, + }, + contents: map[string][]byte{}, + } + + for releaseVersion, releases := range i.Releases { + res.index.Releases[releaseVersion] = release{} + for archiveName := range releases { + var chunk [1024 * 48]byte // 1.5 times our chunk read size in GetVersion + copy(chunk[:], archiveName) + if _, err := rand.Read(chunk[len(archiveName):]); err != nil { + panic(err) + } + content, hash := makeArchive(chunk[:]) + + res.index.Releases[releaseVersion][archiveName] = archive{ + Hash: hash, + // Note: Only storing the name of the archive for now. + // This will be expanded later to a full URL once the server is running. + SelfLink: archiveName, + } + res.contents[archiveName] = content + } + } + return res +} + +func makeArchive(contents []byte) ([]byte, string) { + out := new(bytes.Buffer) + gzipWriter := gzip.NewWriter(out) + tarWriter := tar.NewWriter(gzipWriter) + err := tarWriter.WriteHeader(&tar.Header{ + Name: "controller-tools/envtest/some-file", + Size: int64(len(contents)), + Mode: 0777, // so we can check that we fix this later + }) + if err != nil { + panic(err) + } + _, err = tarWriter.Write(contents) + if err != nil { + panic(err) + } + tarWriter.Close() + gzipWriter.Close() + content := out.Bytes() + // controller-tools is using sha512 + hash := sha512.Sum512(content) + hashEncoded := hex.EncodeToString(hash[:]) + return content, hashEncoded +} diff --git a/pkg/envtest/crd.go b/pkg/envtest/crd.go index f9c58ea26a..8ed2224cfe 100644 --- a/pkg/envtest/crd.go +++ b/pkg/envtest/crd.go @@ -38,7 +38,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/yaml" "sigs.k8s.io/controller-runtime/pkg/client" @@ -94,7 +94,7 @@ func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensio defaultCRDOptions(&options) // Read the CRD yamls into options.CRDs - if err := readCRDFiles(&options); err != nil { + if err := ReadCRDFiles(&options); err != nil { return nil, fmt.Errorf("unable to read CRD files: %w", err) } @@ -115,8 +115,8 @@ func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensio return options.CRDs, nil } -// readCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs. -func readCRDFiles(options *CRDInstallOptions) error { +// ReadCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs. +func ReadCRDFiles(options *CRDInstallOptions) error { if len(options.Paths) > 0 { crdList, err := renderCRDs(options) if err != nil { @@ -217,7 +217,7 @@ func (p *poller) poll(ctx context.Context) (done bool, err error) { // UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory. func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error { // Read the CRD yamls into options.CRDs - if err := readCRDFiles(&options); err != nil { + if err := ReadCRDFiles(&options); err != nil { return err } @@ -229,7 +229,6 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error { // Uninstall each CRD for _, crd := range options.CRDs { - crd := crd log.V(1).Info("uninstalling CRD", "crd", crd.GetName()) if err := cs.Delete(context.TODO(), crd); err != nil { // If CRD is not found, we can consider success @@ -251,7 +250,6 @@ func CreateCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefin // Create each CRD for _, crd := range crds { - crd := crd log.V(1).Info("installing CRD", "crd", crd.GetName()) existingCrd := crd.DeepCopy() err := cs.Get(context.TODO(), client.ObjectKey{Name: crd.GetName()}, existingCrd) @@ -364,19 +362,26 @@ func modifyConversionWebhooks(crds []*apiextensionsv1.CustomResourceDefinition, if err != nil { return err } - url := pointer.String(fmt.Sprintf("https://%s/convert", hostPort)) + url := ptr.To(fmt.Sprintf("https://%s/convert", hostPort)) for i := range crds { // Continue if we're preserving unknown fields. if crds[i].Spec.PreserveUnknownFields { continue } - // Continue if the GroupKind isn't registered as being convertible. - if _, ok := convertibles[schema.GroupKind{ - Group: crds[i].Spec.Group, - Kind: crds[i].Spec.Names.Kind, - }]; !ok { - continue + if !webhookOptions.IgnoreSchemeConvertible { + // Continue if the GroupKind isn't registered as being convertible, + // and remove any existing conversion webhooks if they exist. + // This is to prevent the CRD from being rejected by the apiserver, usually + // manifests that are generated by controller-gen will have a conversion + // webhook set, but we don't want to enable it if the type isn't registered. + if _, ok := convertibles[schema.GroupKind{ + Group: crds[i].Spec.Group, + Kind: crds[i].Spec.Names.Kind, + }]; !ok { + crds[i].Spec.Conversion = nil + continue + } } if crds[i].Spec.Conversion == nil { crds[i].Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ diff --git a/pkg/envtest/crd_test.go b/pkg/envtest/crd_test.go index 92dc48e963..a1406615d6 100644 --- a/pkg/envtest/crd_test.go +++ b/pkg/envtest/crd_test.go @@ -31,7 +31,7 @@ var _ = Describe("Test", func() { "testdata/crdv1_original", }, } - err := readCRDFiles(&opt) + err := ReadCRDFiles(&opt) Expect(err).NotTo(HaveOccurred()) expectedCRDs := sets.NewString( diff --git a/pkg/envtest/envtest_test.go b/pkg/envtest/envtest_test.go index 21464e10be..ce3e9a4d3f 100644 --- a/pkg/envtest/envtest_test.go +++ b/pkg/envtest/envtest_test.go @@ -17,7 +17,6 @@ limitations under the License. package envtest import ( - "context" "path/filepath" "time" @@ -28,7 +27,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" - + "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -55,30 +54,29 @@ var _ = Describe("Test", func() { }) // Cleanup CRDs - AfterEach(func() { + AfterEach(func(ctx SpecContext) { for _, crd := range crds { - crd := crd // Delete only if CRD exists. crdObjectKey := client.ObjectKey{ Name: crd.GetName(), } var placeholder apiextensionsv1.CustomResourceDefinition - if err = c.Get(context.TODO(), crdObjectKey, &placeholder); err != nil && + if err = c.Get(ctx, crdObjectKey, &placeholder); err != nil && apierrors.IsNotFound(err) { // CRD doesn't need to be deleted. continue } Expect(err).NotTo(HaveOccurred()) - Expect(c.Delete(context.TODO(), crd)).To(Succeed()) + Expect(c.Delete(ctx, crd)).To(Succeed()) Eventually(func() bool { - err := c.Get(context.TODO(), crdObjectKey, &placeholder) + err := c.Get(ctx, crdObjectKey, &placeholder) return apierrors.IsNotFound(err) }, 5*time.Second).Should(BeTrue()) } }, teardownTimeoutSeconds) Describe("InstallCRDs", func() { - It("should install the unserved CRDs into the cluster", func() { + It("should install the unserved CRDs into the cluster", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata", "crds", "examplecrd_unserved.yaml")}, }) @@ -87,7 +85,7 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "frigates.ship.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "frigates.ship.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Frigate")) @@ -116,7 +114,7 @@ var _ = Describe("Test", func() { ) Expect(err).NotTo(HaveOccurred()) }) - It("should install the CRDs into the cluster using directory", func() { + It("should install the CRDs into the cluster using directory", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{validDirectory}, }) @@ -125,27 +123,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -245,14 +243,14 @@ var _ = Describe("Test", func() { Expect(err).NotTo(HaveOccurred()) }) - It("should install the CRDs into the cluster using file", func() { + It("should install the CRDs into the cluster using file", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata", "crds", "examplecrd3.yaml")}, }) Expect(err).NotTo(HaveOccurred()) crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "configs.foo.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "configs.foo.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Config")) @@ -291,7 +289,7 @@ var _ = Describe("Test", func() { Expect(crds).To(HaveLen(2)) }) - It("should filter out already existent CRD", func() { + It("should filter out already existent CRD", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{ filepath.Join(".", "testdata"), @@ -301,7 +299,7 @@ var _ = Describe("Test", func() { Expect(err).NotTo(HaveOccurred()) crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) @@ -423,7 +421,7 @@ var _ = Describe("Test", func() { Expect(err).To(HaveOccurred()) }) - It("should reinstall the CRDs if already present in the cluster", func() { + It("should reinstall the CRDs if already present in the cluster", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata")}, @@ -433,27 +431,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -562,27 +560,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -683,7 +681,15 @@ var _ = Describe("Test", func() { }) }) - It("should update CRDs if already present in the cluster", func() { + It("should set a working KubeConfig", func(ctx SpecContext) { + kubeconfigRESTConfig, err := clientcmd.RESTConfigFromKubeConfig(env.KubeConfig) + Expect(err).ToNot(HaveOccurred()) + kubeconfigClient, err := client.New(kubeconfigRESTConfig, client.Options{Scheme: s}) + Expect(err).NotTo(HaveOccurred()) + Expect(kubeconfigClient.List(ctx, &apiextensionsv1.CustomResourceDefinitionList{})).To(Succeed()) + }) + + It("should update CRDs if already present in the cluster", func(ctx SpecContext) { // Install only the CRDv1 multi-version example crds, err = InstallCRDs(env.Config, CRDInstallOptions{ @@ -694,7 +700,7 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) Expect(len(crd.Spec.Versions)).To(BeEquivalentTo(2)) @@ -736,7 +742,7 @@ var _ = Describe("Test", func() { // Expect to find updated CRD crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) Expect(len(crd.Spec.Versions)).To(BeEquivalentTo(3)) @@ -774,8 +780,7 @@ var _ = Describe("Test", func() { }) Describe("UninstallCRDs", func() { - It("should uninstall the CRDs from the cluster", func() { - + It("should uninstall the CRDs from the cluster", func(ctx SpecContext) { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{validDirectory}, }) @@ -784,27 +789,27 @@ var _ = Describe("Test", func() { // Expect to find the CRDs crd := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) crd = &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + err = c.Get(ctx, types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -920,7 +925,7 @@ var _ = Describe("Test", func() { placeholder := &apiextensionsv1.CustomResourceDefinition{} Eventually(func() bool { for _, crd := range crds { - err = c.Get(context.TODO(), types.NamespacedName{Name: crd}, placeholder) + err = c.Get(ctx, types.NamespacedName{Name: crd}, placeholder) notFound := err != nil && apierrors.IsNotFound(err) if !notFound { return false diff --git a/pkg/envtest/komega/OWNERS b/pkg/envtest/komega/OWNERS index ba347dae2b..45f63b0e2e 100644 --- a/pkg/envtest/komega/OWNERS +++ b/pkg/envtest/komega/OWNERS @@ -4,11 +4,10 @@ approvers: - controller-runtime-approvers - schrej - JoelSpeed - - sbueringer reviewers: - controller-runtime-admins - - controller-runtime-reviewers + - controller-runtime-maintainers - controller-runtime-approvers + - controller-runtime-reviewers - schrej - JoelSpeed - - sbueringer diff --git a/pkg/envtest/komega/default.go b/pkg/envtest/komega/default.go index 48fb927a20..dad1f551ae 100644 --- a/pkg/envtest/komega/default.go +++ b/pkg/envtest/komega/default.go @@ -29,7 +29,7 @@ func checkDefaultClient() { // It can be used with gomega.Eventually() like this // // deployment := appsv1.Deployment{ ... } -// gomega.Eventually(komega.Get(&deployment)).To(gomega.Succeed()) +// gomega.Eventually(komega.Get(&deployment)).Should(gomega.Succeed()) // // By calling the returned function directly it can also be used with gomega.Expect(komega.Get(...)()).To(...) func Get(obj client.Object) func() error { @@ -41,7 +41,7 @@ func Get(obj client.Object) func() error { // It can be used with gomega.Eventually() like this // // deployments := v1.DeploymentList{ ... } -// gomega.Eventually(k.List(&deployments)).To(gomega.Succeed()) +// gomega.Eventually(k.List(&deployments)).Should(gomega.Succeed()) // // By calling the returned function directly it can also be used as gomega.Expect(k.List(...)()).To(...) func List(list client.ObjectList, opts ...client.ListOption) func() error { @@ -53,10 +53,9 @@ func List(list client.ObjectList, opts ...client.ListOption) func() error { // It can be used with gomega.Eventually() like this: // // deployment := appsv1.Deployment{ ... } -// gomega.Eventually(k.Update(&deployment, func (o client.Object) { +// gomega.Eventually(k.Update(&deployment, func() { // deployment.Spec.Replicas = 3 -// return &deployment -// })).To(gomega.Succeed()) +// })).Should(gomega.Succeed()) // // By calling the returned function directly it can also be used as gomega.Expect(k.Update(...)()).To(...) func Update(obj client.Object, f func(), opts ...client.UpdateOption) func() error { @@ -68,10 +67,9 @@ func Update(obj client.Object, f func(), opts ...client.UpdateOption) func() err // It can be used with gomega.Eventually() like this: // // deployment := appsv1.Deployment{ ... } -// gomega.Eventually(k.UpdateStatus(&deployment, func (o client.Object) { +// gomega.Eventually(k.UpdateStatus(&deployment, func() { // deployment.Status.AvailableReplicas = 1 -// return &deployment -// })).To(gomega.Succeed()) +// })).Should(gomega.Succeed()) // // By calling the returned function directly it can also be used as gomega.Expect(k.UpdateStatus(...)()).To(...) func UpdateStatus(obj client.Object, f func(), opts ...client.SubResourceUpdateOption) func() error { @@ -83,7 +81,7 @@ func UpdateStatus(obj client.Object, f func(), opts ...client.SubResourceUpdateO // It can be used with gomega.Eventually() like this: // // deployment := appsv1.Deployment{ ... } -// gomega.Eventually(k.Object(&deployment)).To(HaveField("Spec.Replicas", gomega.Equal(pointer.Int32(3)))) +// gomega.Eventually(k.Object(&deployment)).Should(HaveField("Spec.Replicas", gomega.Equal(ptr.To(3)))) // // By calling the returned function directly it can also be used as gomega.Expect(k.Object(...)()).To(...) func Object(obj client.Object) func() (client.Object, error) { @@ -95,7 +93,7 @@ func Object(obj client.Object) func() (client.Object, error) { // It can be used with gomega.Eventually() like this: // // deployments := appsv1.DeploymentList{ ... } -// gomega.Eventually(k.ObjectList(&deployments)).To(HaveField("Items", HaveLen(1))) +// gomega.Eventually(k.ObjectList(&deployments)).Should(HaveField("Items", HaveLen(1))) // // By calling the returned function directly it can also be used as gomega.Expect(k.ObjectList(...)()).To(...) func ObjectList(list client.ObjectList, opts ...client.ListOption) func() (client.ObjectList, error) { diff --git a/pkg/envtest/komega/default_test.go b/pkg/envtest/komega/default_test.go index 238a4abd9e..1a1de72cf3 100644 --- a/pkg/envtest/komega/default_test.go +++ b/pkg/envtest/komega/default_test.go @@ -6,7 +6,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestDefaultGet(t *testing.T) { @@ -95,7 +95,7 @@ func TestDefaultObject(t *testing.T) { } g.Eventually(Object(&fetched)).Should(And( Not(BeNil()), - HaveField("Spec.Replicas", Equal(pointer.Int32(5))), + HaveField("Spec.Replicas", Equal(ptr.To(int32(5)))), )) } @@ -110,7 +110,7 @@ func TestDefaultObjectList(t *testing.T) { Not(BeNil()), HaveField("Items", And( HaveLen(1), - ContainElement(HaveField("Spec.Replicas", Equal(pointer.Int32(5)))), + ContainElement(HaveField("Spec.Replicas", Equal(ptr.To(int32(5))))), )), )) } diff --git a/pkg/envtest/komega/interfaces.go b/pkg/envtest/komega/interfaces.go index 6f7e5db35b..b412e5c1bf 100644 --- a/pkg/envtest/komega/interfaces.go +++ b/pkg/envtest/komega/interfaces.go @@ -42,9 +42,8 @@ type Komega interface { // Update returns a function that fetches a resource, applies the provided update function and then updates the resource. // It can be used with gomega.Eventually() like this: // deployment := appsv1.Deployment{ ... } - // gomega.Eventually(k.Update(&deployment, func (o client.Object) { + // gomega.Eventually(k.Update(&deployment, func() { // deployment.Spec.Replicas = 3 - // return &deployment // })).To(gomega.Succeed()) // By calling the returned function directly it can also be used as gomega.Expect(k.Update(...)()).To(...) Update(client.Object, func(), ...client.UpdateOption) func() error @@ -52,9 +51,8 @@ type Komega interface { // UpdateStatus returns a function that fetches a resource, applies the provided update function and then updates the resource's status. // It can be used with gomega.Eventually() like this: // deployment := appsv1.Deployment{ ... } - // gomega.Eventually(k.Update(&deployment, func (o client.Object) { + // gomega.Eventually(k.Update(&deployment, func() { // deployment.Status.AvailableReplicas = 1 - // return &deployment // })).To(gomega.Succeed()) // By calling the returned function directly it can also be used as gomega.Expect(k.UpdateStatus(...)()).To(...) UpdateStatus(client.Object, func(), ...client.SubResourceUpdateOption) func() error @@ -62,7 +60,7 @@ type Komega interface { // Object returns a function that fetches a resource and returns the object. // It can be used with gomega.Eventually() like this: // deployment := appsv1.Deployment{ ... } - // gomega.Eventually(k.Object(&deployment)).To(HaveField("Spec.Replicas", gomega.Equal(pointer.Int32(3)))) + // gomega.Eventually(k.Object(&deployment)).To(HaveField("Spec.Replicas", gomega.Equal(ptr.To(int32(3))))) // By calling the returned function directly it can also be used as gomega.Expect(k.Object(...)()).To(...) Object(client.Object) func() (client.Object, error) diff --git a/pkg/envtest/komega/komega_test.go b/pkg/envtest/komega/komega_test.go index 275610c8bb..8867ac239a 100644 --- a/pkg/envtest/komega/komega_test.go +++ b/pkg/envtest/komega/komega_test.go @@ -7,7 +7,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -20,7 +20,7 @@ func exampleDeployment() *appsv1.Deployment { Name: "test", }, Spec: appsv1.DeploymentSpec{ - Replicas: pointer.Int32(5), + Replicas: ptr.To(int32(5)), }, } } @@ -117,7 +117,7 @@ func TestObject(t *testing.T) { } g.Eventually(k.Object(&fetched)).Should(And( Not(BeNil()), - HaveField("Spec.Replicas", Equal(pointer.Int32(5))), + HaveField("Spec.Replicas", Equal(ptr.To(int32(5)))), )) } @@ -132,7 +132,7 @@ func TestObjectList(t *testing.T) { Not(BeNil()), HaveField("Items", And( HaveLen(1), - ContainElement(HaveField("Spec.Replicas", Equal(pointer.Int32(5)))), + ContainElement(HaveField("Spec.Replicas", Equal(ptr.To(int32(5))))), )), )) } diff --git a/pkg/envtest/server.go b/pkg/envtest/server.go index 4ee440df9c..9bb81ed2ab 100644 --- a/pkg/envtest/server.go +++ b/pkg/envtest/server.go @@ -17,15 +17,20 @@ limitations under the License. package envtest import ( + "context" "fmt" "os" "strings" "time" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" @@ -121,6 +126,10 @@ type Environment struct { // loading. Config *rest.Config + // KubeConfig provides []byte of a kubeconfig file to talk to the apiserver + // It's automatically populated if not set based on the `Config` + KubeConfig []byte + // CRDInstallOptions are the options for installing CRDs. CRDInstallOptions CRDInstallOptions @@ -142,8 +151,22 @@ type Environment struct { // values are merged. CRDDirectoryPaths []string + // DownloadBinaryAssets indicates that the envtest binaries should be downloaded. + // If BinaryAssetsDirectory is also set, it is used to store the downloaded binaries, + // otherwise a tmp directory is created. + DownloadBinaryAssets bool + + // DownloadBinaryAssetsVersion is the version of envtest binaries to download. + // Defaults to the latest stable version (i.e. excluding alpha / beta / RC versions). + DownloadBinaryAssetsVersion string + + // DownloadBinaryAssetsIndexURL is the index used to discover envtest binaries to download. + // Defaults to https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml. + DownloadBinaryAssetsIndexURL string + // BinaryAssetsDirectory is the path where the binaries required for the envtest are // located in the local environment. This field can be overridden by setting KUBEBUILDER_ASSETS. + // Set this field to SetupEnvtestDefaultBinaryAssetsDirectory() to share binaries with setup-envtest. BinaryAssetsDirectory string // UseExistingCluster indicates that this environments should use an @@ -161,11 +184,6 @@ type Environment struct { // environment variable or 20 seconds if unspecified ControlPlaneStopTimeout time.Duration - // KubeAPIServerFlags is the set of flags passed while starting the api server. - // - // Deprecated: use ControlPlane.GetAPIServer().Configure() instead. - KubeAPIServerFlags []string - // AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr. // Enable this to get more visibility of the testing control plane. // It respect KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT environment variable. @@ -210,19 +228,7 @@ func (te *Environment) Start() (*rest.Config, error) { } } else { apiServer := te.ControlPlane.GetAPIServer() - if len(apiServer.Args) == 0 { //nolint:staticcheck - // pass these through separately from above in case something like - // AddUser defaults APIServer. - // - // TODO(directxman12): if/when we feel like making a bigger - // breaking change here, just make APIServer and Etcd non-pointers - // in ControlPlane. - - // NB(directxman12): we still pass these in so that things work if the - // user manually specifies them, but in most cases we expect them to - // be nil so that we use the new .Configure() logic. - apiServer.Args = te.KubeAPIServerFlags //nolint:staticcheck - } + if te.ControlPlane.Etcd == nil { te.ControlPlane.Etcd = &controlplane.Etcd{} } @@ -245,9 +251,21 @@ func (te *Environment) Start() (*rest.Config, error) { } } - apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory) - te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory) - te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory) + if te.DownloadBinaryAssets { + apiServerPath, etcdPath, kubectlPath, err := downloadBinaryAssets(context.TODO(), + te.BinaryAssetsDirectory, te.DownloadBinaryAssetsVersion, te.DownloadBinaryAssetsIndexURL) + if err != nil { + return nil, err + } + + apiServer.Path = apiServerPath + te.ControlPlane.Etcd.Path = etcdPath + te.ControlPlane.KubectlPath = kubectlPath + } else { + apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory) + te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory) + te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory) + } if err := te.defaultTimeouts(); err != nil { return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err) @@ -277,11 +295,25 @@ func (te *Environment) Start() (*rest.Config, error) { te.Config = adminUser.Config() } + if len(te.KubeConfig) == 0 { + var err error + te.KubeConfig, err = controlplane.KubeConfigFromREST(te.Config) + if err != nil { + return nil, fmt.Errorf("unable to set KubeConfig field: %w", err) + } + } + // Set the default scheme if nil. if te.Scheme == nil { te.Scheme = scheme.Scheme } + // If we are bringing etcd up for the first time, it can take some time for the + // default namespace to actually be created and seen as available to the apiserver + if err := te.waitForDefaultNamespace(te.Config); err != nil { + return nil, fmt.Errorf("default namespace didn't register within deadline: %w", err) + } + // Call PrepWithoutInstalling to setup certificates first // and have them available to patch CRD conversion webhook as well. if err := te.WebhookInstallOptions.PrepWithoutInstalling(); err != nil { @@ -289,6 +321,9 @@ func (te *Environment) Start() (*rest.Config, error) { } log.V(1).Info("installing CRDs") + if te.CRDInstallOptions.Scheme == nil { + te.CRDInstallOptions.Scheme = te.Scheme + } te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs) te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths) te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing @@ -336,6 +371,20 @@ func (te *Environment) startControlPlane() error { return nil } +func (te *Environment) waitForDefaultNamespace(config *rest.Config) error { + cs, err := client.New(config, client.Options{}) + if err != nil { + return fmt.Errorf("unable to create client: %w", err) + } + // It shouldn't take longer than 5s for the default namespace to be brought up in etcd + return wait.PollUntilContextTimeout(context.TODO(), time.Millisecond*50, time.Second*5, true, func(ctx context.Context) (bool, error) { + if err = cs.Get(ctx, types.NamespacedName{Name: "default"}, &corev1.Namespace{}); err != nil { + return false, nil //nolint:nilerr + } + return true, nil + }) +} + func (te *Environment) defaultTimeouts() error { var err error if te.ControlPlaneStartTimeout == 0 { diff --git a/pkg/envtest/webhook.go b/pkg/envtest/webhook.go index f7e43a1480..a6961bf7c6 100644 --- a/pkg/envtest/webhook.go +++ b/pkg/envtest/webhook.go @@ -49,6 +49,11 @@ type WebhookInstallOptions struct { // ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install ValidatingWebhooks []*admissionv1.ValidatingWebhookConfiguration + // IgnoreSchemeConvertible, will modify any CRD conversion webhook to use the local serving host and port, + // bypassing the need to have the types registered in the Scheme. This is useful for testing CRD conversion webhooks + // with unregistered or unstructured types. + IgnoreSchemeConvertible bool + // IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true IgnoreErrorIfPathMissing bool @@ -184,7 +189,8 @@ func defaultWebhookOptions(o *WebhookInstallOptions) { func WaitForWebhooks(config *rest.Config, mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration, validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration, - options WebhookInstallOptions) error { + options WebhookInstallOptions, +) error { waitingFor := map[schema.GroupVersionKind]*sets.Set[string]{} for _, hook := range mutatingWebhooks { @@ -242,7 +248,7 @@ func (p *webhookPoller) poll(ctx context.Context) (done bool, err error) { continue } for _, name := range names.UnsortedList() { - var obj = &unstructured.Unstructured{} + obj := &unstructured.Unstructured{} obj.SetGroupVersionKind(gvk) err := c.Get(context.Background(), client.ObjectKey{ Namespace: "", @@ -288,10 +294,10 @@ func (o *WebhookInstallOptions) setupCA() error { return fmt.Errorf("unable to marshal webhook serving certs: %w", err) } - if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { return fmt.Errorf("unable to write webhook serving cert to disk: %w", err) } - if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { return fmt.Errorf("unable to write webhook serving key to disk: %w", err) } @@ -307,14 +313,12 @@ func createWebhooks(config *rest.Config, mutHooks []*admissionv1.MutatingWebhook // Create each webhook for _, hook := range mutHooks { - hook := hook log.V(1).Info("installing mutating webhook", "webhook", hook.GetName()) if err := ensureCreated(cs, hook); err != nil { return err } } for _, hook := range valHooks { - hook := hook log.V(1).Info("installing validating webhook", "webhook", hook.GetName()) if err := ensureCreated(cs, hook); err != nil { return err @@ -415,8 +419,8 @@ func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []* const ( admissionregv1 = "admissionregistration.k8s.io/v1" ) - switch { - case generic.Kind == "MutatingWebhookConfiguration": + switch generic.Kind { + case "MutatingWebhookConfiguration": if generic.APIVersion != admissionregv1 { return nil, nil, fmt.Errorf("only v1 is supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name) } @@ -425,7 +429,7 @@ func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []* return nil, nil, err } mutHooks = append(mutHooks, hook) - case generic.Kind == "ValidatingWebhookConfiguration": + case "ValidatingWebhookConfiguration": if generic.APIVersion != admissionregv1 { return nil, nil, fmt.Errorf("only v1 is supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name) } diff --git a/pkg/envtest/webhook_test.go b/pkg/envtest/webhook_test.go index 3a87052580..47550fa147 100644 --- a/pkg/envtest/webhook_test.go +++ b/pkg/envtest/webhook_test.go @@ -29,6 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -36,22 +37,21 @@ import ( ) var _ = Describe("Test", func() { - Describe("Webhook", func() { - It("should reject create request for webhook that rejects all requests", func() { + It("should reject create request for webhook that rejects all requests", func(specCtx SpecContext) { m, err := manager.New(env.Config, manager.Options{ - Port: env.WebhookInstallOptions.LocalServingPort, - Host: env.WebhookInstallOptions.LocalServingHost, - CertDir: env.WebhookInstallOptions.LocalServingCertDir, - TLSOpts: []func(*tls.Config){ - func(config *tls.Config) {}, - }, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: env.WebhookInstallOptions.LocalServingPort, + Host: env.WebhookInstallOptions.LocalServingHost, + CertDir: env.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + }), }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { _ = server.Start(ctx) }() @@ -87,7 +87,7 @@ var _ = Describe("Test", func() { } Eventually(func() bool { - err = c.Create(context.TODO(), obj) + err = c.Create(ctx, obj) return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) diff --git a/pkg/event/event.go b/pkg/event/event.go index 271b3c00fb..82b1793f53 100644 --- a/pkg/event/event.go +++ b/pkg/event/event.go @@ -18,38 +18,58 @@ package event import "sigs.k8s.io/controller-runtime/pkg/client" -// CreateEvent is an event where a Kubernetes object was created. CreateEvent should be generated +// CreateEvent is an event where a Kubernetes object was created. CreateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by a handler.EventHandler. +type CreateEvent = TypedCreateEvent[client.Object] + +// UpdateEvent is an event where a Kubernetes object was updated. UpdateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. +type UpdateEvent = TypedUpdateEvent[client.Object] + +// DeleteEvent is an event where a Kubernetes object was deleted. DeleteEvent should be generated // by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. -type CreateEvent struct { +type DeleteEvent = TypedDeleteEvent[client.Object] + +// GenericEvent is an event where the operation type is unknown (e.g. polling or event originating outside the cluster). +// GenericEvent should be generated by a source.Source and transformed into a reconcile.Request by an +// handler.EventHandler. +type GenericEvent = TypedGenericEvent[client.Object] + +// TypedCreateEvent is an event where a Kubernetes object was created. TypedCreateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.TypedEventHandler. +type TypedCreateEvent[object any] struct { // Object is the object from the event - Object client.Object + Object object + + // IsInInitialList is true if the Create event was triggered by the initial list. + IsInInitialList bool } -// UpdateEvent is an event where a Kubernetes object was updated. UpdateEvent should be generated -// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. -type UpdateEvent struct { +// TypedUpdateEvent is an event where a Kubernetes object was updated. TypedUpdateEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.TypedEventHandler. +type TypedUpdateEvent[object any] struct { // ObjectOld is the object from the event - ObjectOld client.Object + ObjectOld object // ObjectNew is the object from the event - ObjectNew client.Object + ObjectNew object } -// DeleteEvent is an event where a Kubernetes object was deleted. DeleteEvent should be generated -// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler. -type DeleteEvent struct { +// TypedDeleteEvent is an event where a Kubernetes object was deleted. TypedDeleteEvent should be generated +// by a source.Source and transformed into a reconcile.Request by an handler.TypedEventHandler. +type TypedDeleteEvent[object any] struct { // Object is the object from the event - Object client.Object + Object object // DeleteStateUnknown is true if the Delete event was missed but we identified the object // as having been deleted. DeleteStateUnknown bool } -// GenericEvent is an event where the operation type is unknown (e.g. polling or event originating outside the cluster). -// GenericEvent should be generated by a source.Source and transformed into a reconcile.Request by an -// handler.EventHandler. -type GenericEvent struct { +// TypedGenericEvent is an event where the operation type is unknown (e.g. polling or event originating outside the cluster). +// TypedGenericEvent should be generated by a source.Source and transformed into a reconcile.Request by an +// handler.TypedEventHandler. +type TypedGenericEvent[object any] struct { // Object is the object from the event - Object client.Object + Object object } diff --git a/pkg/finalizer/finalizer_test.go b/pkg/finalizer/finalizer_test.go index 02fbdf003c..c6848f6473 100644 --- a/pkg/finalizer/finalizer_test.go +++ b/pkg/finalizer/finalizer_test.go @@ -57,14 +57,14 @@ var _ = Describe("TestFinalizer", func() { }) Describe("Finalize", func() { - It("successfully finalizes and returns true for Updated when deletion timestamp is nil and finalizer does not exist", func() { + It("successfully finalizes and returns true for Updated when deletion timestamp is nil and finalizer does not exist", func(ctx SpecContext) { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) Expect(err).ToNot(HaveOccurred()) pod.DeletionTimestamp = nil pod.Finalizers = []string{} - result, err := finalizers.Finalize(context.TODO(), pod) + result, err := finalizers.Finalize(ctx, pod) Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) // when deletion timestamp is nil and finalizer is not present, the registered finalizer would be added to the obj @@ -73,7 +73,7 @@ var _ = Describe("TestFinalizer", func() { }) - It("successfully finalizes and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func() { + It("successfully finalizes and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now @@ -82,37 +82,37 @@ var _ = Describe("TestFinalizer", func() { pod.Finalizers = []string{"finalizers.sigs.k8s.io/testfinalizer"} - result, err := finalizers.Finalize(context.TODO(), pod) + result, err := finalizers.Finalize(ctx, pod) Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) // finalizer will be removed from the obj upon successful finalization Expect(pod.Finalizers).To(BeEmpty()) }) - It("should return no error and return false for Updated when deletion timestamp is nil and finalizer doesn't exist", func() { + It("should return no error and return false for Updated when deletion timestamp is nil and finalizer doesn't exist", func(ctx SpecContext) { pod.DeletionTimestamp = nil pod.Finalizers = []string{} - result, err := finalizers.Finalize(context.TODO(), pod) + result, err := finalizers.Finalize(ctx, pod) Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeFalse()) Expect(pod.Finalizers).To(BeEmpty()) }) - It("should return no error and return false for Updated when deletion timestamp is not nil and the finalizer doesn't exist", func() { + It("should return no error and return false for Updated when deletion timestamp is not nil and the finalizer doesn't exist", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now pod.Finalizers = []string{} - result, err := finalizers.Finalize(context.TODO(), pod) + result, err := finalizers.Finalize(ctx, pod) Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeFalse()) Expect(pod.Finalizers).To(BeEmpty()) }) - It("successfully finalizes multiple finalizers and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func() { + It("successfully finalizes multiple finalizers and returns true for Updated when deletion timestamp is not nil and the finalizer exists", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now @@ -124,14 +124,14 @@ var _ = Describe("TestFinalizer", func() { pod.Finalizers = []string{"finalizers.sigs.k8s.io/testfinalizer", "finalizers.sigs.k8s.io/newtestfinalizer"} - result, err := finalizers.Finalize(context.TODO(), pod) + result, err := finalizers.Finalize(ctx, pod) Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) Expect(result.StatusUpdated).To(BeFalse()) Expect(pod.Finalizers).To(BeEmpty()) }) - It("should return result as false and a non-nil error", func() { + It("should return result as false and a non-nil error", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now pod.Finalizers = []string{"finalizers.sigs.k8s.io/testfinalizer"} @@ -143,7 +143,7 @@ var _ = Describe("TestFinalizer", func() { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer", f) Expect(err).ToNot(HaveOccurred()) - result, err := finalizers.Finalize(context.TODO(), pod) + result, err := finalizers.Finalize(ctx, pod) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("finalizer failed")) Expect(result.Updated).To(BeFalse()) @@ -152,7 +152,7 @@ var _ = Describe("TestFinalizer", func() { Expect(pod.Finalizers[0]).To(Equal("finalizers.sigs.k8s.io/testfinalizer")) }) - It("should return expected result values and error values when registering multiple finalizers", func() { + It("should return expected result values and error values when registering multiple finalizers", func(ctx SpecContext) { now := metav1.Now() pod.DeletionTimestamp = &now pod.Finalizers = []string{ @@ -169,7 +169,7 @@ var _ = Describe("TestFinalizer", func() { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer1", f) Expect(err).ToNot(HaveOccurred()) - result, err := finalizers.Finalize(context.TODO(), pod) + result, err := finalizers.Finalize(ctx, pod) Expect(err).ToNot(HaveOccurred()) Expect(result.Updated).To(BeTrue()) Expect(result.StatusUpdated).To(BeFalse()) @@ -186,7 +186,7 @@ var _ = Describe("TestFinalizer", func() { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer2", f) Expect(err).ToNot(HaveOccurred()) - result, err = finalizers.Finalize(context.TODO(), pod) + result, err = finalizers.Finalize(ctx, pod) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("finalizer failed")) Expect(result.Updated).To(BeFalse()) @@ -202,7 +202,7 @@ var _ = Describe("TestFinalizer", func() { err = finalizers.Register("finalizers.sigs.k8s.io/testfinalizer3", f) Expect(err).ToNot(HaveOccurred()) - result, err = finalizers.Finalize(context.TODO(), pod) + result, err = finalizers.Finalize(ctx, pod) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("finalizer failed")) Expect(result.Updated).To(BeTrue()) diff --git a/pkg/handler/enqueue.go b/pkg/handler/enqueue.go index c72b2e1ebb..64cbe8a4d1 100644 --- a/pkg/handler/enqueue.go +++ b/pkg/handler/enqueue.go @@ -18,9 +18,11 @@ package handler import ( "context" + "reflect" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -33,43 +35,57 @@ type empty struct{} var _ EventHandler = &EnqueueRequestForObject{} // EnqueueRequestForObject enqueues a Request containing the Name and Namespace of the object that is the source of the Event. -// (e.g. the created / deleted / updated objects Name and Namespace). handler.EnqueueRequestForObject is used by almost all +// (e.g. the created / deleted / updated objects Name and Namespace). handler.EnqueueRequestForObject is used by almost all // Controllers that have associated Resources (e.g. CRDs) to reconcile the associated Resource. -type EnqueueRequestForObject struct{} +type EnqueueRequestForObject = TypedEnqueueRequestForObject[client.Object] + +// TypedEnqueueRequestForObject enqueues a Request containing the Name and Namespace of the object that is the source of the Event. +// (e.g. the created / deleted / updated objects Name and Namespace). handler.TypedEnqueueRequestForObject is used by almost all +// Controllers that have associated Resources (e.g. CRDs) to reconcile the associated Resource. +// +// TypedEnqueueRequestForObject is experimental and subject to future change. +type TypedEnqueueRequestForObject[object client.Object] struct{} // Create implements EventHandler. -func (e *EnqueueRequestForObject) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { - if evt.Object == nil { +func (e *TypedEnqueueRequestForObject[T]) Create(ctx context.Context, evt event.TypedCreateEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if isNil(evt.Object) { enqueueLog.Error(nil, "CreateEvent received with no metadata", "event", evt) return } - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + + item := reconcile.Request{NamespacedName: types.NamespacedName{ Name: evt.Object.GetName(), Namespace: evt.Object.GetNamespace(), - }}) + }} + + addToQueueCreate(q, evt, item) } // Update implements EventHandler. -func (e *EnqueueRequestForObject) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *TypedEnqueueRequestForObject[T]) Update(ctx context.Context, evt event.TypedUpdateEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { switch { - case evt.ObjectNew != nil: - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + case !isNil(evt.ObjectNew): + item := reconcile.Request{NamespacedName: types.NamespacedName{ Name: evt.ObjectNew.GetName(), Namespace: evt.ObjectNew.GetNamespace(), - }}) - case evt.ObjectOld != nil: - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + }} + + addToQueueUpdate(q, evt, item) + case !isNil(evt.ObjectOld): + item := reconcile.Request{NamespacedName: types.NamespacedName{ Name: evt.ObjectOld.GetName(), Namespace: evt.ObjectOld.GetNamespace(), - }}) + }} + + addToQueueUpdate(q, evt, item) default: enqueueLog.Error(nil, "UpdateEvent received with no metadata", "event", evt) } } // Delete implements EventHandler. -func (e *EnqueueRequestForObject) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { - if evt.Object == nil { +func (e *TypedEnqueueRequestForObject[T]) Delete(ctx context.Context, evt event.TypedDeleteEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if isNil(evt.Object) { enqueueLog.Error(nil, "DeleteEvent received with no metadata", "event", evt) return } @@ -80,8 +96,8 @@ func (e *EnqueueRequestForObject) Delete(ctx context.Context, evt event.DeleteEv } // Generic implements EventHandler. -func (e *EnqueueRequestForObject) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { - if evt.Object == nil { +func (e *TypedEnqueueRequestForObject[T]) Generic(ctx context.Context, evt event.TypedGenericEvent[T], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if isNil(evt.Object) { enqueueLog.Error(nil, "GenericEvent received with no metadata", "event", evt) return } @@ -90,3 +106,15 @@ func (e *EnqueueRequestForObject) Generic(ctx context.Context, evt event.Generic Namespace: evt.Object.GetNamespace(), }}) } + +func isNil(arg any) bool { + if v := reflect.ValueOf(arg); !v.IsValid() || ((v.Kind() == reflect.Ptr || + v.Kind() == reflect.Interface || + v.Kind() == reflect.Slice || + v.Kind() == reflect.Map || + v.Kind() == reflect.Chan || + v.Kind() == reflect.Func) && v.IsNil()) { + return true + } + return false +} diff --git a/pkg/handler/enqueue_mapped.go b/pkg/handler/enqueue_mapped.go index b55fdde6ba..62d6728151 100644 --- a/pkg/handler/enqueue_mapped.go +++ b/pkg/handler/enqueue_mapped.go @@ -20,14 +20,22 @@ import ( "context" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // MapFunc is the signature required for enqueueing requests from a generic function. // This type is usually used with EnqueueRequestsFromMapFunc when registering an event handler. -type MapFunc func(context.Context, client.Object) []reconcile.Request +type MapFunc = TypedMapFunc[client.Object, reconcile.Request] + +// TypedMapFunc is the signature required for enqueueing requests from a generic function. +// This type is usually used with EnqueueRequestsFromTypedMapFunc when registering an event handler. +// +// TypedMapFunc is experimental and subject to future change. +type TypedMapFunc[object any, request comparable] func(context.Context, object) []request // EnqueueRequestsFromMapFunc enqueues Requests by running a transformation function that outputs a collection // of reconcile.Requests on each Event. The reconcile.Requests may be for an arbitrary set of objects @@ -40,48 +48,105 @@ type MapFunc func(context.Context, client.Object) []reconcile.Request // For UpdateEvents which contain both a new and old object, the transformation function is run on both // objects and both sets of Requests are enqueue. func EnqueueRequestsFromMapFunc(fn MapFunc) EventHandler { - return &enqueueRequestsFromMapFunc{ - toRequests: fn, + return TypedEnqueueRequestsFromMapFunc(fn) +} + +// TypedEnqueueRequestsFromMapFunc enqueues Requests by running a transformation function that outputs a collection +// of reconcile.Requests on each Event. The reconcile.Requests may be for an arbitrary set of objects +// defined by some user specified transformation of the source Event. (e.g. trigger Reconciler for a set of objects +// in response to a cluster resize event caused by adding or deleting a Node) +// +// TypedEnqueueRequestsFromMapFunc is frequently used to fan-out updates from one object to one or more other +// objects of a differing type. +// +// For TypedUpdateEvents which contain both a new and old object, the transformation function is run on both +// objects and both sets of Requests are enqueue. +// +// TypedEnqueueRequestsFromMapFunc is experimental and subject to future change. +func TypedEnqueueRequestsFromMapFunc[object any, request comparable](fn TypedMapFunc[object, request]) TypedEventHandler[object, request] { + return &enqueueRequestsFromMapFunc[object, request]{ + toRequests: fn, + objectImplementsClientObject: implementsClientObject[object](), } } -var _ EventHandler = &enqueueRequestsFromMapFunc{} +var _ EventHandler = &enqueueRequestsFromMapFunc[client.Object, reconcile.Request]{} -type enqueueRequestsFromMapFunc struct { +type enqueueRequestsFromMapFunc[object any, request comparable] struct { // Mapper transforms the argument into a slice of keys to be reconciled - toRequests MapFunc + toRequests TypedMapFunc[object, request] + objectImplementsClientObject bool } // Create implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(ctx, q, evt.Object, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Create( + ctx context.Context, + evt event.TypedCreateEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + reqs := map[request]empty{} + + var lowPriority bool + if isPriorityQueue(q) && !isNil(evt.Object) { + if evt.IsInInitialList { + lowPriority = true + } + } + e.mapAndEnqueue(ctx, q, evt.Object, reqs, lowPriority) } // Update implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(ctx, q, evt.ObjectOld, reqs) - e.mapAndEnqueue(ctx, q, evt.ObjectNew, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Update( + ctx context.Context, + evt event.TypedUpdateEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + var lowPriority bool + if e.objectImplementsClientObject && isPriorityQueue(q) && !isNil(evt.ObjectOld) && !isNil(evt.ObjectNew) { + lowPriority = any(evt.ObjectOld).(client.Object).GetResourceVersion() == any(evt.ObjectNew).(client.Object).GetResourceVersion() + } + reqs := map[request]empty{} + e.mapAndEnqueue(ctx, q, evt.ObjectOld, reqs, lowPriority) + e.mapAndEnqueue(ctx, q, evt.ObjectNew, reqs, lowPriority) } // Delete implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(ctx, q, evt.Object, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Delete( + ctx context.Context, + evt event.TypedDeleteEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + reqs := map[request]empty{} + e.mapAndEnqueue(ctx, q, evt.Object, reqs, false) } // Generic implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { - reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(ctx, q, evt.Object, reqs) +func (e *enqueueRequestsFromMapFunc[object, request]) Generic( + ctx context.Context, + evt event.TypedGenericEvent[object], + q workqueue.TypedRateLimitingInterface[request], +) { + reqs := map[request]empty{} + e.mapAndEnqueue(ctx, q, evt.Object, reqs, false) } -func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(ctx context.Context, q workqueue.RateLimitingInterface, object client.Object, reqs map[reconcile.Request]empty) { - for _, req := range e.toRequests(ctx, object) { +func (e *enqueueRequestsFromMapFunc[object, request]) mapAndEnqueue( + ctx context.Context, + q workqueue.TypedRateLimitingInterface[request], + o object, + reqs map[request]empty, + lowPriority bool, +) { + for _, req := range e.toRequests(ctx, o) { _, ok := reqs[req] if !ok { - q.Add(req) + if lowPriority { + q.(priorityqueue.PriorityQueue[request]).AddWithOpts(priorityqueue.AddOpts{ + Priority: ptr.To(LowPriority), + }, req) + } else { + q.Add(req) + } reqs[req] = empty{} } } diff --git a/pkg/handler/enqueue_owner.go b/pkg/handler/enqueue_owner.go index 02e7d756f8..e8fc8eb46e 100644 --- a/pkg/handler/enqueue_owner.go +++ b/pkg/handler/enqueue_owner.go @@ -32,12 +32,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -var _ EventHandler = &enqueueRequestForOwner{} +var _ EventHandler = &enqueueRequestForOwner[client.Object]{} var log = logf.RuntimeLog.WithName("eventhandler").WithName("enqueueRequestForOwner") // OwnerOption modifies an EnqueueRequestForOwner EventHandler. -type OwnerOption func(e *enqueueRequestForOwner) +type OwnerOption func(e enqueueRequestForOwnerInterface) // EnqueueRequestForOwner enqueues Requests for the Owners of an object. E.g. the object that created // the object that was the source of the Event. @@ -48,7 +48,21 @@ type OwnerOption func(e *enqueueRequestForOwner) // // - a handler.enqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and OnlyControllerOwner set to true. func EnqueueRequestForOwner(scheme *runtime.Scheme, mapper meta.RESTMapper, ownerType client.Object, opts ...OwnerOption) EventHandler { - e := &enqueueRequestForOwner{ + return TypedEnqueueRequestForOwner[client.Object](scheme, mapper, ownerType, opts...) +} + +// TypedEnqueueRequestForOwner enqueues Requests for the Owners of an object. E.g. the object that created +// the object that was the source of the Event. +// +// If a ReplicaSet creates Pods, users may reconcile the ReplicaSet in response to Pod Events using: +// +// - a source.Kind Source with Type of Pod. +// +// - a handler.typedEnqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and OnlyControllerOwner set to true. +// +// TypedEnqueueRequestForOwner is experimental and subject to future change. +func TypedEnqueueRequestForOwner[object client.Object](scheme *runtime.Scheme, mapper meta.RESTMapper, ownerType client.Object, opts ...OwnerOption) TypedEventHandler[object, reconcile.Request] { + e := &enqueueRequestForOwner[object]{ ownerType: ownerType, mapper: mapper, } @@ -58,17 +72,21 @@ func EnqueueRequestForOwner(scheme *runtime.Scheme, mapper meta.RESTMapper, owne for _, opt := range opts { opt(e) } - return e + return WithLowPriorityWhenUnchanged(e) } // OnlyControllerOwner if provided will only look at the first OwnerReference with Controller: true. func OnlyControllerOwner() OwnerOption { - return func(e *enqueueRequestForOwner) { - e.isController = true + return func(e enqueueRequestForOwnerInterface) { + e.setIsController(true) } } -type enqueueRequestForOwner struct { +type enqueueRequestForOwnerInterface interface { + setIsController(bool) +} + +type enqueueRequestForOwner[object client.Object] struct { // ownerType is the type of the Owner object to look for in OwnerReferences. Only Group and Kind are compared. ownerType runtime.Object @@ -82,8 +100,12 @@ type enqueueRequestForOwner struct { mapper meta.RESTMapper } +func (e *enqueueRequestForOwner[object]) setIsController(isController bool) { + e.isController = isController +} + // Create implements EventHandler. -func (e *enqueueRequestForOwner) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Create(ctx context.Context, evt event.TypedCreateEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -92,7 +114,7 @@ func (e *enqueueRequestForOwner) Create(ctx context.Context, evt event.CreateEve } // Update implements EventHandler. -func (e *enqueueRequestForOwner) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Update(ctx context.Context, evt event.TypedUpdateEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.ObjectOld, reqs) e.getOwnerReconcileRequest(evt.ObjectNew, reqs) @@ -102,7 +124,7 @@ func (e *enqueueRequestForOwner) Update(ctx context.Context, evt event.UpdateEve } // Delete implements EventHandler. -func (e *enqueueRequestForOwner) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Delete(ctx context.Context, evt event.TypedDeleteEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -111,7 +133,7 @@ func (e *enqueueRequestForOwner) Delete(ctx context.Context, evt event.DeleteEve } // Generic implements EventHandler. -func (e *enqueueRequestForOwner) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner[object]) Generic(ctx context.Context, evt event.TypedGenericEvent[object], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -121,7 +143,7 @@ func (e *enqueueRequestForOwner) Generic(ctx context.Context, evt event.GenericE // parseOwnerTypeGroupKind parses the OwnerType into a Group and Kind and caches the result. Returns false // if the OwnerType could not be parsed using the scheme. -func (e *enqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error { +func (e *enqueueRequestForOwner[object]) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error { // Get the kinds of the type kinds, _, err := scheme.ObjectKinds(e.ownerType) if err != nil { @@ -141,10 +163,10 @@ func (e *enqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) // getOwnerReconcileRequest looks at object and builds a map of reconcile.Request to reconcile // owners of object that match e.OwnerType. -func (e *enqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, result map[reconcile.Request]empty) { +func (e *enqueueRequestForOwner[object]) getOwnerReconcileRequest(obj metav1.Object, result map[reconcile.Request]empty) { // Iterate through the OwnerReferences looking for a match on Group and Kind against what was requested // by the user - for _, ref := range e.getOwnersReferences(object) { + for _, ref := range e.getOwnersReferences(obj) { // Parse the Group out of the OwnerReference to compare it to what was parsed out of the requested OwnerType refGV, err := schema.ParseGroupVersion(ref.APIVersion) if err != nil { @@ -170,7 +192,7 @@ func (e *enqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, return } if mapping.Scope.Name() != meta.RESTScopeNameRoot { - request.Namespace = object.GetNamespace() + request.Namespace = obj.GetNamespace() } result[request] = empty{} @@ -181,17 +203,17 @@ func (e *enqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, // getOwnersReferences returns the OwnerReferences for an object as specified by the enqueueRequestForOwner // - if IsController is true: only take the Controller OwnerReference (if found) // - if IsController is false: take all OwnerReferences. -func (e *enqueueRequestForOwner) getOwnersReferences(object metav1.Object) []metav1.OwnerReference { - if object == nil { +func (e *enqueueRequestForOwner[object]) getOwnersReferences(obj metav1.Object) []metav1.OwnerReference { + if obj == nil { return nil } // If not filtered as Controller only, then use all the OwnerReferences if !e.isController { - return object.GetOwnerReferences() + return obj.GetOwnerReferences() } // If filtered to a Controller, only take the Controller OwnerReference - if ownerRef := metav1.GetControllerOf(object); ownerRef != nil { + if ownerRef := metav1.GetControllerOf(obj); ownerRef != nil { return []metav1.OwnerReference{*ownerRef} } // No Controller OwnerReference found diff --git a/pkg/handler/eventhandler.go b/pkg/handler/eventhandler.go index 2f380f4fc4..88510d29ed 100644 --- a/pkg/handler/eventhandler.go +++ b/pkg/handler/eventhandler.go @@ -18,14 +18,20 @@ package handler import ( "context" + "reflect" + "time" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // EventHandler enqueues reconcile.Requests in response to events (e.g. Pod Create). EventHandlers map an Event // for one object to trigger Reconciles for either the same object or different objects - e.g. if there is an -// Event for object with type Foo (using source.KindSource) then reconcile one or more object(s) with type Bar. +// Event for object with type Foo (using source.Kind) then reconcile one or more object(s) with type Bar. // // Identical reconcile.Requests will be batched together through the queuing mechanism before reconcile is called. // @@ -41,66 +47,201 @@ import ( // // Unless you are implementing your own EventHandler, you can ignore the functions on the EventHandler interface. // Most users shouldn't need to implement their own EventHandler. -type EventHandler interface { - // Create is called in response to an create event - e.g. Pod Creation. - Create(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) +type EventHandler = TypedEventHandler[client.Object, reconcile.Request] + +// TypedEventHandler enqueues reconcile.Requests in response to events (e.g. Pod Create). TypedEventHandlers map an Event +// for one object to trigger Reconciles for either the same object or different objects - e.g. if there is an +// Event for object with type Foo (using source.Kind) then reconcile one or more object(s) with type Bar. +// +// Identical reconcile.Requests will be batched together through the queuing mechanism before reconcile is called. +// +// * Use TypedEnqueueRequestForObject to reconcile the object the event is for +// - do this for events for the type the Controller Reconciles. (e.g. Deployment for a Deployment Controller) +// +// * Use TypedEnqueueRequestForOwner to reconcile the owner of the object the event is for +// - do this for events for the types the Controller creates. (e.g. ReplicaSets created by a Deployment Controller) +// +// * Use TypedEnqueueRequestsFromMapFunc to transform an event for an object to a reconcile of an object +// of a different type - do this for events for types the Controller may be interested in, but doesn't create. +// (e.g. If Foo responds to cluster size events, map Node events to Foo objects.) +// +// Unless you are implementing your own TypedEventHandler, you can ignore the functions on the TypedEventHandler interface. +// Most users shouldn't need to implement their own TypedEventHandler. +// +// TypedEventHandler is experimental and subject to future change. +type TypedEventHandler[object any, request comparable] interface { + // Create is called in response to a create event - e.g. Pod Creation. + Create(context.Context, event.TypedCreateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Update is called in response to an update event - e.g. Pod Updated. - Update(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) + Update(context.Context, event.TypedUpdateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Delete is called in response to a delete event - e.g. Pod Deleted. - Delete(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) + Delete(context.Context, event.TypedDeleteEvent[object], workqueue.TypedRateLimitingInterface[request]) // Generic is called in response to an event of an unknown type or a synthetic event triggered as a cron or // external trigger request - e.g. reconcile Autoscaling, or a Webhook. - Generic(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) + Generic(context.Context, event.TypedGenericEvent[object], workqueue.TypedRateLimitingInterface[request]) } var _ EventHandler = Funcs{} -// Funcs implements EventHandler. -type Funcs struct { +// Funcs implements eventhandler. +type Funcs = TypedFuncs[client.Object, reconcile.Request] + +// TypedFuncs implements eventhandler. +// +// TypedFuncs is experimental and subject to future change. +type TypedFuncs[object any, request comparable] struct { // Create is called in response to an add event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - CreateFunc func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) + CreateFunc func(context.Context, event.TypedCreateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Update is called in response to an update event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - UpdateFunc func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) + UpdateFunc func(context.Context, event.TypedUpdateEvent[object], workqueue.TypedRateLimitingInterface[request]) // Delete is called in response to a delete event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - DeleteFunc func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) + DeleteFunc func(context.Context, event.TypedDeleteEvent[object], workqueue.TypedRateLimitingInterface[request]) // GenericFunc is called in response to a generic event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - GenericFunc func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) + GenericFunc func(context.Context, event.TypedGenericEvent[object], workqueue.TypedRateLimitingInterface[request]) +} + +var typeForClientObject = reflect.TypeFor[client.Object]() + +func implementsClientObject[object any]() bool { + return reflect.TypeFor[object]().Implements(typeForClientObject) +} + +func isPriorityQueue[request comparable](q workqueue.TypedRateLimitingInterface[request]) bool { + _, ok := q.(priorityqueue.PriorityQueue[request]) + return ok } // Create implements EventHandler. -func (h Funcs) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Create(ctx context.Context, e event.TypedCreateEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.CreateFunc != nil { - h.CreateFunc(ctx, e, q) + if !implementsClientObject[object]() || !isPriorityQueue(q) || isNil(e.Object) { + h.CreateFunc(ctx, e, q) + return + } + + wq := workqueueWithDefaultPriority[request]{ + // We already know that we have a priority queue, that event.Object implements + // client.Object and that its not nil + PriorityQueue: q.(priorityqueue.PriorityQueue[request]), + } + if e.IsInInitialList { + wq.priority = ptr.To(LowPriority) + } + h.CreateFunc(ctx, e, wq) } } // Delete implements EventHandler. -func (h Funcs) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Delete(ctx context.Context, e event.TypedDeleteEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.DeleteFunc != nil { h.DeleteFunc(ctx, e, q) } } // Update implements EventHandler. -func (h Funcs) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Update(ctx context.Context, e event.TypedUpdateEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.UpdateFunc != nil { - h.UpdateFunc(ctx, e, q) + if !implementsClientObject[object]() || !isPriorityQueue(q) || isNil(e.ObjectOld) || isNil(e.ObjectNew) { + h.UpdateFunc(ctx, e, q) + return + } + + wq := workqueueWithDefaultPriority[request]{ + // We already know that we have a priority queue, that event.ObjectOld and ObjectNew implement + // client.Object and that they are not nil + PriorityQueue: q.(priorityqueue.PriorityQueue[request]), + } + if any(e.ObjectOld).(client.Object).GetResourceVersion() == any(e.ObjectNew).(client.Object).GetResourceVersion() { + wq.priority = ptr.To(LowPriority) + } + h.UpdateFunc(ctx, e, wq) } } // Generic implements EventHandler. -func (h Funcs) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h TypedFuncs[object, request]) Generic(ctx context.Context, e event.TypedGenericEvent[object], q workqueue.TypedRateLimitingInterface[request]) { if h.GenericFunc != nil { h.GenericFunc(ctx, e, q) } } + +// LowPriority is the priority set by WithLowPriorityWhenUnchanged +const LowPriority = -100 + +// WithLowPriorityWhenUnchanged reduces the priority of events stemming from the initial listwatch or from a resync if +// and only if a priorityqueue.PriorityQueue is used. If not, it does nothing. +func WithLowPriorityWhenUnchanged[object client.Object, request comparable](u TypedEventHandler[object, request]) TypedEventHandler[object, request] { + // TypedFuncs already implements this so just wrap + return TypedFuncs[object, request]{ + CreateFunc: u.Create, + UpdateFunc: u.Update, + DeleteFunc: u.Delete, + GenericFunc: u.Generic, + } +} + +type workqueueWithDefaultPriority[request comparable] struct { + priorityqueue.PriorityQueue[request] + priority *int +} + +func (w workqueueWithDefaultPriority[request]) Add(item request) { + w.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: w.priority}, item) +} + +func (w workqueueWithDefaultPriority[request]) AddAfter(item request, after time.Duration) { + w.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: w.priority, After: after}, item) +} + +func (w workqueueWithDefaultPriority[request]) AddRateLimited(item request) { + w.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: w.priority, RateLimited: true}, item) +} + +func (w workqueueWithDefaultPriority[request]) AddWithOpts(o priorityqueue.AddOpts, items ...request) { + if o.Priority == nil { + o.Priority = w.priority + } + w.PriorityQueue.AddWithOpts(o, items...) +} + +// addToQueueCreate adds the reconcile.Request to the priorityqueue in the handler +// for Create requests if and only if the workqueue being used is of type priorityqueue.PriorityQueue[reconcile.Request] +func addToQueueCreate[T client.Object, request comparable](q workqueue.TypedRateLimitingInterface[request], evt event.TypedCreateEvent[T], item request) { + priorityQueue, isPriorityQueue := q.(priorityqueue.PriorityQueue[request]) + if !isPriorityQueue { + q.Add(item) + return + } + + var priority *int + if evt.IsInInitialList { + priority = ptr.To(LowPriority) + } + priorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: priority}, item) +} + +// addToQueueUpdate adds the reconcile.Request to the priorityqueue in the handler +// for Update requests if and only if the workqueue being used is of type priorityqueue.PriorityQueue[reconcile.Request] +func addToQueueUpdate[T client.Object, request comparable](q workqueue.TypedRateLimitingInterface[request], evt event.TypedUpdateEvent[T], item request) { + priorityQueue, isPriorityQueue := q.(priorityqueue.PriorityQueue[request]) + if !isPriorityQueue { + q.Add(item) + return + } + + var priority *int + if evt.ObjectOld.GetResourceVersion() == evt.ObjectNew.GetResourceVersion() { + priority = ptr.To(LowPriority) + } + priorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: priority}, item) +} diff --git a/pkg/handler/eventhandler_test.go b/pkg/handler/eventhandler_test.go index 6ae87b0988..2a7453f761 100644 --- a/pkg/handler/eventhandler_test.go +++ b/pkg/handler/eventhandler_test.go @@ -18,6 +18,7 @@ package handler_test import ( "context" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,23 +31,23 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) var _ = Describe("Eventhandler", func() { - var ctx = context.Background() - var q workqueue.RateLimitingInterface + var q workqueue.TypedRateLimitingInterface[reconcile.Request] var instance handler.EnqueueRequestForObject var pod *corev1.Pod var mapper meta.RESTMapper BeforeEach(func() { - q = &controllertest.Queue{Interface: workqueue.New()} + q = &controllertest.Queue{TypedInterface: workqueue.NewTyped[reconcile.Request]()} pod = &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Namespace: "biz", Name: "baz"}, } @@ -54,41 +55,35 @@ var _ = Describe("Eventhandler", func() { httpClient, err := rest.HTTPClientFor(cfg) Expect(err).ShouldNot(HaveOccurred()) - mapper, err = apiutil.NewDiscoveryRESTMapper(cfg, httpClient) + mapper, err = apiutil.NewDynamicRESTMapper(cfg, httpClient) Expect(err).ShouldNot(HaveOccurred()) }) Describe("EnqueueRequestForObject", func() { - It("should enqueue a Request with the Name / Namespace of the object in the CreateEvent.", func() { + It("should enqueue a Request with the Name / Namespace of the object in the CreateEvent.", func(ctx SpecContext) { evt := event.CreateEvent{ Object: pod, } instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) - It("should enqueue a Request with the Name / Namespace of the object in the DeleteEvent.", func() { + It("should enqueue a Request with the Name / Namespace of the object in the DeleteEvent.", func(ctx SpecContext) { evt := event.DeleteEvent{ Object: pod, } instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) It("should enqueue a Request with the Name / Namespace of one object in the UpdateEvent.", - func() { + func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = "baz2" newPod.Namespace = "biz2" @@ -100,28 +95,22 @@ var _ = Describe("Eventhandler", func() { instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz2", Name: "baz2"})) }) - It("should enqueue a Request with the Name / Namespace of the object in the GenericEvent.", func() { + It("should enqueue a Request with the Name / Namespace of the object in the GenericEvent.", func(ctx SpecContext) { evt := event.GenericEvent{ Object: pod, } instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) Context("for a runtime.Object without Object", func() { - It("should do nothing if the Object is missing for a CreateEvent.", func() { + It("should do nothing if the Object is missing for a CreateEvent.", func(ctx SpecContext) { evt := event.CreateEvent{ Object: nil, } @@ -129,7 +118,7 @@ var _ = Describe("Eventhandler", func() { Expect(q.Len()).To(Equal(0)) }) - It("should do nothing if the Object is missing for a UpdateEvent.", func() { + It("should do nothing if the Object is missing for a UpdateEvent.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = "baz2" newPod.Namespace = "biz2" @@ -140,24 +129,18 @@ var _ = Describe("Eventhandler", func() { } instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ := q.Get() - Expect(i).NotTo(BeNil()) - req, ok := i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ := q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz2", Name: "baz2"})) evt.ObjectNew = nil evt.ObjectOld = pod instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) - i, _ = q.Get() - Expect(i).NotTo(BeNil()) - req, ok = i.(reconcile.Request) - Expect(ok).To(BeTrue()) + req, _ = q.Get() Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) }) - It("should do nothing if the Object is missing for a DeleteEvent.", func() { + It("should do nothing if the Object is missing for a DeleteEvent.", func(ctx SpecContext) { evt := event.DeleteEvent{ Object: nil, } @@ -165,7 +148,7 @@ var _ = Describe("Eventhandler", func() { Expect(q.Len()).To(Equal(0)) }) - It("should do nothing if the Object is missing for a GenericEvent.", func() { + It("should do nothing if the Object is missing for a GenericEvent.", func(ctx SpecContext) { evt := event.GenericEvent{ Object: nil, } @@ -176,7 +159,7 @@ var _ = Describe("Eventhandler", func() { }) Describe("EnqueueRequestsFromMapFunc", func() { - It("should enqueue a Request with the function applied to the CreateEvent.", func() { + It("should enqueue a Request with the function applied to the CreateEvent.", func(ctx SpecContext) { req := []reconcile.Request{} instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() @@ -208,7 +191,7 @@ var _ = Describe("Eventhandler", func() { )) }) - It("should enqueue a Request with the function applied to the DeleteEvent.", func() { + It("should enqueue a Request with the function applied to the DeleteEvent.", func(ctx SpecContext) { req := []reconcile.Request{} instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() @@ -241,7 +224,7 @@ var _ = Describe("Eventhandler", func() { }) It("should enqueue a Request with the function applied to both objects in the UpdateEvent.", - func() { + func(ctx SpecContext) { newPod := pod.DeepCopy() req := []reconcile.Request{} @@ -273,7 +256,7 @@ var _ = Describe("Eventhandler", func() { Expect(i).To(Equal(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "biz", Name: "baz-baz"}})) }) - It("should enqueue a Request with the function applied to the GenericEvent.", func() { + It("should enqueue a Request with the function applied to the GenericEvent.", func(ctx SpecContext) { req := []reconcile.Request{} instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() @@ -307,7 +290,7 @@ var _ = Describe("Eventhandler", func() { }) Describe("EnqueueRequestForOwner", func() { - It("should enqueue a Request with the Owner of the object in the CreateEvent.", func() { + It("should enqueue a Request with the Owner of the object in the CreateEvent.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ @@ -328,7 +311,7 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request with the Owner of the object in the DeleteEvent.", func() { + It("should enqueue a Request with the Owner of the object in the DeleteEvent.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ @@ -349,7 +332,7 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request with the Owners of both objects in the UpdateEvent.", func() { + It("should enqueue a Request with the Owners of both objects in the UpdateEvent.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" @@ -387,7 +370,7 @@ var _ = Describe("Eventhandler", func() { )) }) - It("should enqueue a Request with the one duplicate Owner of both objects in the UpdateEvent.", func() { + It("should enqueue a Request with the one duplicate Owner of both objects in the UpdateEvent.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" @@ -419,7 +402,7 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request with the Owner of the object in the GenericEvent.", func() { + It("should enqueue a Request with the Owner of the object in the GenericEvent.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -439,7 +422,7 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should not enqueue a Request if there are no owners matching Group and Kind.", func() { + It("should not enqueue a Request if there are no owners matching Group and Kind.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { // Wrong group @@ -461,13 +444,13 @@ var _ = Describe("Eventhandler", func() { }) It("should enqueue a Request if there are owners matching Group "+ - "and Kind with a different version.", func() { + "and Kind with a different version.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &autoscalingv1.HorizontalPodAutoscaler{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo-parent", Kind: "HorizontalPodAutoscaler", - APIVersion: "autoscaling/v2beta1", + APIVersion: "autoscaling/v2", }, } evt := event.CreateEvent{ @@ -481,7 +464,7 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo-parent"}})) }) - It("should enqueue a Request for a owner that is cluster scoped", func() { + It("should enqueue a Request for a owner that is cluster scoped", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &corev1.Node{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -502,7 +485,7 @@ var _ = Describe("Eventhandler", func() { }) - It("should not enqueue a Request if there are no owners.", func() { + It("should not enqueue a Request if there are no owners.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) evt := event.CreateEvent{ Object: pod, @@ -513,7 +496,7 @@ var _ = Describe("Eventhandler", func() { Context("with the Controller field set to true", func() { It("should enqueue reconcile.Requests for only the first the Controller if there are "+ - "multiple Controller owners.", func() { + "multiple Controller owners.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -525,7 +508,7 @@ var _ = Describe("Eventhandler", func() { Name: "foo2-parent", Kind: "ReplicaSet", APIVersion: "apps/v1", - Controller: pointer.Bool(true), + Controller: ptr.To(true), }, { Name: "foo3-parent", @@ -536,7 +519,7 @@ var _ = Describe("Eventhandler", func() { Name: "foo4-parent", Kind: "ReplicaSet", APIVersion: "apps/v1", - Controller: pointer.Bool(true), + Controller: ptr.To(true), }, { Name: "foo5-parent", @@ -554,7 +537,7 @@ var _ = Describe("Eventhandler", func() { NamespacedName: types.NamespacedName{Namespace: pod.GetNamespace(), Name: "foo2-parent"}})) }) - It("should not enqueue reconcile.Requests if there are no Controller owners.", func() { + It("should not enqueue reconcile.Requests if there are no Controller owners.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -580,7 +563,7 @@ var _ = Describe("Eventhandler", func() { Expect(q.Len()).To(Equal(0)) }) - It("should not enqueue reconcile.Requests if there are no owners.", func() { + It("should not enqueue reconcile.Requests if there are no owners.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) evt := event.CreateEvent{ Object: pod, @@ -591,7 +574,7 @@ var _ = Describe("Eventhandler", func() { }) Context("with the Controller field set to false", func() { - It("should enqueue a reconcile.Requests for all owners.", func() { + It("should enqueue a reconcile.Requests for all owners.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -631,7 +614,7 @@ var _ = Describe("Eventhandler", func() { }) Context("with a nil object", func() { - It("should do nothing.", func() { + It("should do nothing.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -657,7 +640,7 @@ var _ = Describe("Eventhandler", func() { }) Context("with an invalid APIVersion in the OwnerReference", func() { - It("should do nothing.", func() { + It("should do nothing.", func(ctx SpecContext) { instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -676,31 +659,31 @@ var _ = Describe("Eventhandler", func() { }) Describe("Funcs", func() { - failingFuncs := handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { + failingFuncs := handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect CreateEvent to be called.") }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect DeleteEvent to be called.") }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect UpdateEvent to be called.") }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect GenericEvent to be called.") }, } - It("should call CreateFunc for a CreateEvent if provided.", func() { + It("should call CreateFunc for a CreateEvent if provided.", func(ctx SpecContext) { instance := failingFuncs evt := event.CreateEvent{ Object: pod, } - instance.CreateFunc = func(ctx context.Context, evt2 event.CreateEvent, q2 workqueue.RateLimitingInterface) { + instance.CreateFunc = func(ctx context.Context, evt2 event.CreateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) @@ -708,7 +691,7 @@ var _ = Describe("Eventhandler", func() { instance.Create(ctx, evt, q) }) - It("should NOT call CreateFunc for a CreateEvent if NOT provided.", func() { + It("should NOT call CreateFunc for a CreateEvent if NOT provided.", func(ctx SpecContext) { instance := failingFuncs instance.CreateFunc = nil evt := event.CreateEvent{ @@ -717,7 +700,7 @@ var _ = Describe("Eventhandler", func() { instance.Create(ctx, evt, q) }) - It("should call UpdateFunc for an UpdateEvent if provided.", func() { + It("should call UpdateFunc for an UpdateEvent if provided.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" @@ -727,7 +710,7 @@ var _ = Describe("Eventhandler", func() { } instance := failingFuncs - instance.UpdateFunc = func(ctx context.Context, evt2 event.UpdateEvent, q2 workqueue.RateLimitingInterface) { + instance.UpdateFunc = func(ctx context.Context, evt2 event.UpdateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) @@ -736,7 +719,7 @@ var _ = Describe("Eventhandler", func() { instance.Update(ctx, evt, q) }) - It("should NOT call UpdateFunc for an UpdateEvent if NOT provided.", func() { + It("should NOT call UpdateFunc for an UpdateEvent if NOT provided.", func(ctx SpecContext) { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" @@ -747,12 +730,12 @@ var _ = Describe("Eventhandler", func() { instance.Update(ctx, evt, q) }) - It("should call DeleteFunc for a DeleteEvent if provided.", func() { + It("should call DeleteFunc for a DeleteEvent if provided.", func(ctx SpecContext) { instance := failingFuncs evt := event.DeleteEvent{ Object: pod, } - instance.DeleteFunc = func(ctx context.Context, evt2 event.DeleteEvent, q2 workqueue.RateLimitingInterface) { + instance.DeleteFunc = func(ctx context.Context, evt2 event.DeleteEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) @@ -760,7 +743,7 @@ var _ = Describe("Eventhandler", func() { instance.Delete(ctx, evt, q) }) - It("should NOT call DeleteFunc for a DeleteEvent if NOT provided.", func() { + It("should NOT call DeleteFunc for a DeleteEvent if NOT provided.", func(ctx SpecContext) { instance := failingFuncs instance.DeleteFunc = nil evt := event.DeleteEvent{ @@ -769,12 +752,12 @@ var _ = Describe("Eventhandler", func() { instance.Delete(ctx, evt, q) }) - It("should call GenericFunc for a GenericEvent if provided.", func() { + It("should call GenericFunc for a GenericEvent if provided.", func(ctx SpecContext) { instance := failingFuncs evt := event.GenericEvent{ Object: pod, } - instance.GenericFunc = func(ctx context.Context, evt2 event.GenericEvent, q2 workqueue.RateLimitingInterface) { + instance.GenericFunc = func(ctx context.Context, evt2 event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) @@ -782,7 +765,7 @@ var _ = Describe("Eventhandler", func() { instance.Generic(ctx, evt, q) }) - It("should NOT call GenericFunc for a GenericEvent if NOT provided.", func() { + It("should NOT call GenericFunc for a GenericEvent if NOT provided.", func(ctx SpecContext) { instance := failingFuncs instance.GenericFunc = nil evt := event.GenericEvent{ @@ -791,4 +774,395 @@ var _ = Describe("Eventhandler", func() { instance.Generic(ctx, evt, q) }) }) + + Describe("WithLowPriorityWhenUnchanged", func() { + handlerPriorityTests := []struct { + name string + handler func() handler.EventHandler + after time.Duration + ratelimited bool + overridePriority int + }{ + { + name: "WithLowPriorityWhenUnchanged wrapper", + handler: func() handler.EventHandler { return handler.WithLowPriorityWhenUnchanged(customHandler{}) }, + }, + { + name: "EnqueueRequestForObject", + handler: func() handler.EventHandler { return &handler.EnqueueRequestForObject{} }, + }, + { + name: "EnqueueRequestForOwner", + handler: func() handler.EventHandler { + return handler.EnqueueRequestForOwner( + scheme.Scheme, + mapper, + &corev1.Pod{}, + ) + }, + }, + { + name: "TypedEnqueueRequestForOwner", + handler: func() handler.EventHandler { + return handler.TypedEnqueueRequestForOwner[client.Object]( + scheme.Scheme, + mapper, + &corev1.Pod{}, + ) + }, + }, + { + name: "Funcs", + handler: func() handler.EventHandler { + return handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + } + }, + }, + { + name: "EnqueueRequestsFromMapFunc", + handler: func() handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + }}} + }) + }, + }, + { + name: "WithLowPriorityWhenUnchanged - Add", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + }) + }, + }, + { + name: "WithLowPriorityWhenUnchanged - AddAfter", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddAfter(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}, time.Second) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddAfter(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}, time.Second) + }, + }) + }, + after: time.Second, + }, + { + name: "WithLowPriorityWhenUnchanged - AddRateLimited", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddRateLimited(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + wq.AddRateLimited(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + }) + }, + ratelimited: true, + }, + { + name: "WithLowPriorityWhenUnchanged - AddWithOpts priority is retained", + handler: func() handler.EventHandler { + return handler.WithLowPriorityWhenUnchanged( + handler.TypedFuncs[client.Object, reconcile.Request]{ + CreateFunc: func(ctx context.Context, tce event.TypedCreateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if pq, isPQ := wq.(priorityqueue.PriorityQueue[reconcile.Request]); isPQ { + pq.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(100)}, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + return + } + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tce.Object.GetNamespace(), + Name: tce.Object.GetName(), + }}) + }, + UpdateFunc: func(ctx context.Context, tue event.TypedUpdateEvent[client.Object], wq workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if pq, isPQ := wq.(priorityqueue.PriorityQueue[reconcile.Request]); isPQ { + pq.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(100)}, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + return + } + wq.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: tue.ObjectNew.GetNamespace(), + Name: tue.ObjectNew.GetName(), + }}) + }, + }) + }, + overridePriority: 100, + }, + } + for _, test := range handlerPriorityTests { + When("handler is "+test.name, func() { + It("should lower the priority of a create request for an object that was part of the initial list", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Create(ctx, event.CreateEvent{ + Object: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + CreationTimestamp: metav1.Now(), + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + IsInInitialList: true, + }, wq) + + expected := handler.LowPriority + if test.overridePriority != 0 { + expected = test.overridePriority + } + + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{ + Priority: ptr.To(expected), + After: test.after, + RateLimited: test.ratelimited, + })) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should not lower the priority of a create request for an object that was not part of the initial list", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Create(ctx, event.CreateEvent{ + Object: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + CreationTimestamp: metav1.Now(), + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + IsInInitialList: false, + }, wq) + + var expectedPriority *int + if test.overridePriority != 0 { + expectedPriority = &test.overridePriority + } + + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{After: test.after, RateLimited: test.ratelimited, Priority: expectedPriority})) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should lower the priority of an update request with unchanged RV", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Update(ctx, event.UpdateEvent{ + ObjectOld: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + ObjectNew: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, wq) + + expectedPriority := handler.LowPriority + if test.overridePriority != 0 { + expectedPriority = test.overridePriority + } + + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{After: test.after, RateLimited: test.ratelimited, Priority: ptr.To(expectedPriority)})) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should not lower the priority of an update request with changed RV", func(ctx SpecContext) { + actualOpts := priorityqueue.AddOpts{} + var actualRequests []reconcile.Request + wq := &fakePriorityQueue{ + addWithOpts: func(o priorityqueue.AddOpts, items ...reconcile.Request) { + actualOpts = o + actualRequests = items + }, + } + + test.handler().Update(ctx, event.UpdateEvent{ + ObjectOld: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + ObjectNew: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + ResourceVersion: "1", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, wq) + + var expectedPriority *int + if test.overridePriority != 0 { + expectedPriority = &test.overridePriority + } + Expect(actualOpts).To(Equal(priorityqueue.AddOpts{After: test.after, RateLimited: test.ratelimited, Priority: expectedPriority})) + Expect(actualRequests).To(Equal([]reconcile.Request{{NamespacedName: types.NamespacedName{Name: "my-pod"}}})) + }) + + It("should have no effect on create if the workqueue is not a priorityqueue", func(ctx SpecContext) { + test.handler().Create(ctx, event.CreateEvent{ + Object: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, q) + + Expect(q.Len()).To(Equal(1)) + item, _ := q.Get() + Expect(item).To(Equal(reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-pod"}})) + }) + + It("should have no effect on Update if the workqueue is not a priorityqueue", func(ctx SpecContext) { + test.handler().Update(ctx, event.UpdateEvent{ + ObjectOld: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + ObjectNew: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + OwnerReferences: []metav1.OwnerReference{{ + Kind: "Pod", + Name: "my-pod", + }}, + }}, + }, q) + + Expect(q.Len()).To(Equal(1)) + item, _ := q.Get() + Expect(item).To(Equal(reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-pod"}})) + }) + }) + } + }) }) + +type fakePriorityQueue struct { + workqueue.TypedRateLimitingInterface[reconcile.Request] + addWithOpts func(o priorityqueue.AddOpts, items ...reconcile.Request) +} + +func (f *fakePriorityQueue) Add(item reconcile.Request) { + f.AddWithOpts(priorityqueue.AddOpts{}, item) +} + +func (f *fakePriorityQueue) AddWithOpts(o priorityqueue.AddOpts, items ...reconcile.Request) { + f.addWithOpts(o, items...) +} +func (f *fakePriorityQueue) GetWithPriority() (item reconcile.Request, priority int, shutdown bool) { + panic("GetWithPriority is not expected to be called") +} + +// customHandler re-implements the basic enqueueRequestForObject logic +// to be able to test the WithLowPriorityWhenUnchanged wrapper +type customHandler struct{} + +func (ch customHandler) Create(ctx context.Context, evt event.CreateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.Object.GetNamespace(), + Name: evt.Object.GetName(), + }}) +} +func (ch customHandler) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.ObjectNew.GetNamespace(), + Name: evt.ObjectNew.GetName(), + }}) +} +func (ch customHandler) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.Object.GetNamespace(), + Name: evt.Object.GetName(), + }}) +} +func (ch customHandler) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: evt.Object.GetNamespace(), + Name: evt.Object.GetName(), + }}) +} diff --git a/pkg/handler/example_test.go b/pkg/handler/example_test.go index ad07e4e31d..ad87e4be63 100644 --- a/pkg/handler/example_test.go +++ b/pkg/handler/example_test.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -32,16 +31,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) -var mgr manager.Manager -var c controller.Controller +var ( + mgr manager.Manager + c controller.Controller +) // This example watches Pods and enqueues Requests with the Name and Namespace of the Pod from // the Event (i.e. change caused by a Create, Update, Delete). func ExampleEnqueueRequestForObject() { // controller is a controller.controller err := c.Watch( - source.Kind(mgr.GetCache(), &corev1.Pod{}), - &handler.EnqueueRequestForObject{}, + source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), ) if err != nil { // handle it @@ -53,8 +53,10 @@ func ExampleEnqueueRequestForObject() { func ExampleEnqueueRequestForOwner() { // controller is a controller.controller err := c.Watch( - source.Kind(mgr.GetCache(), &appsv1.ReplicaSet{}), - handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), + source.Kind(mgr.GetCache(), + &appsv1.ReplicaSet{}, + handler.TypedEnqueueRequestForOwner[*appsv1.ReplicaSet](mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), + ), ) if err != nil { // handle it @@ -66,19 +68,20 @@ func ExampleEnqueueRequestForOwner() { func ExampleEnqueueRequestsFromMapFunc() { // controller is a controller.controller err := c.Watch( - source.Kind(mgr.GetCache(), &appsv1.Deployment{}), - handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { - return []reconcile.Request{ - {NamespacedName: types.NamespacedName{ - Name: a.GetName() + "-1", - Namespace: a.GetNamespace(), - }}, - {NamespacedName: types.NamespacedName{ - Name: a.GetName() + "-2", - Namespace: a.GetNamespace(), - }}, - } - }), + source.Kind(mgr.GetCache(), &appsv1.Deployment{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, a *appsv1.Deployment) []reconcile.Request { + return []reconcile.Request{ + {NamespacedName: types.NamespacedName{ + Name: a.Name + "-1", + Namespace: a.Namespace, + }}, + {NamespacedName: types.NamespacedName{ + Name: a.Name + "-2", + Namespace: a.Namespace, + }}, + } + }), + ), ) if err != nil { // handle it @@ -89,33 +92,34 @@ func ExampleEnqueueRequestsFromMapFunc() { func ExampleFuncs() { // controller is a controller.controller err := c.Watch( - source.Kind(mgr.GetCache(), &corev1.Pod{}), - handler.Funcs{ - CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.Object.GetName(), - Namespace: e.Object.GetNamespace(), - }}) - }, - UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.ObjectNew.GetName(), - Namespace: e.ObjectNew.GetNamespace(), - }}) - }, - DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.Object.GetName(), - Namespace: e.Object.GetNamespace(), - }}) - }, - GenericFunc: func(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { - q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ - Name: e.Object.GetName(), - Namespace: e.Object.GetNamespace(), - }}) + source.Kind(mgr.GetCache(), &corev1.Pod{}, + handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(ctx context.Context, e event.TypedCreateEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.Object.Name, + Namespace: e.Object.Namespace, + }}) + }, + UpdateFunc: func(ctx context.Context, e event.TypedUpdateEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.ObjectNew.Name, + Namespace: e.ObjectNew.Namespace, + }}) + }, + DeleteFunc: func(ctx context.Context, e event.TypedDeleteEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.Object.Name, + Namespace: e.Object.Namespace, + }}) + }, + GenericFunc: func(ctx context.Context, e event.TypedGenericEvent[*corev1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: e.Object.Name, + Namespace: e.Object.Namespace, + }}) + }, }, - }, + ), ) if err != nil { // handle it diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go index 83aba28cb7..7dd06957eb 100644 --- a/pkg/internal/controller/controller.go +++ b/pkg/internal/controller/controller.go @@ -21,23 +21,75 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" "github.com/go-logr/logr" + "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/handler" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics" logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) +// Options are the arguments for creating a new Controller. +type Options[request comparable] struct { + // Reconciler is a function that can be called at any time with the Name / Namespace of an object and + // ensures that the state of the system matches the state specified in the object. + // Defaults to the DefaultReconcileFunc. + Do reconcile.TypedReconciler[request] + + // RateLimiter is used to limit how frequently requests may be queued into the work queue. + RateLimiter workqueue.TypedRateLimiter[request] + + // NewQueue constructs the queue for this controller once the controller is ready to start. + // This is a func because the standard Kubernetes work queues start themselves immediately, which + // leads to goroutine leaks if something calls controller.New repeatedly. + NewQueue func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. + MaxConcurrentReconciles int + + // CacheSyncTimeout refers to the time limit set on waiting for cache to sync + // Defaults to 2 minutes if not set. + CacheSyncTimeout time.Duration + + // Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required. + Name string + + // LogConstructor is used to construct a logger to then log messages to users during reconciliation, + // or for example when a watch is started. + // Note: LogConstructor has to be able to handle nil requests as we are also using it + // outside the context of a reconciliation. + LogConstructor func(request *request) logr.Logger + + // RecoverPanic indicates whether the panic caused by reconcile should be recovered. + // Defaults to true. + RecoverPanic *bool + + // LeaderElected indicates whether the controller is leader elected or always running. + LeaderElected *bool + + // EnableWarmup specifies whether the controller should start its sources + // when the manager is not the leader. + // Defaults to false, which means that the controller will wait for leader election to start + // before starting sources. + EnableWarmup *bool + + // ReconciliationTimeout is used as the timeout passed to the context of each Reconcile call. + // By default, there is no timeout. + ReconciliationTimeout time.Duration +} + // Controller implements controller.Controller. -type Controller struct { +type Controller[request comparable] struct { // Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required. Name string @@ -47,16 +99,19 @@ type Controller struct { // Reconciler is a function that can be called at any time with the Name / Namespace of an object and // ensures that the state of the system matches the state specified in the object. // Defaults to the DefaultReconcileFunc. - Do reconcile.Reconciler + Do reconcile.TypedReconciler[request] - // MakeQueue constructs the queue for this controller once the controller is ready to start. - // This exists because the standard Kubernetes workqueues start themselves immediately, which + // RateLimiter is used to limit how frequently requests may be queued into the work queue. + RateLimiter workqueue.TypedRateLimiter[request] + + // NewQueue constructs the queue for this controller once the controller is ready to start. + // This is a func because the standard Kubernetes work queues start themselves immediately, which // leads to goroutine leaks if something calls controller.New repeatedly. - MakeQueue func() workqueue.RateLimitingInterface + NewQueue func(controllerName string, rateLimiter workqueue.TypedRateLimiter[request]) workqueue.TypedRateLimitingInterface[request] // Queue is an listeningQueue that listens for events from Informers and adds object keys to // the Queue for processing - Queue workqueue.RateLimitingInterface + Queue priorityqueue.PriorityQueue[request] // mu is used to synchronize Controller setup mu sync.Mutex @@ -76,35 +131,71 @@ type Controller struct { CacheSyncTimeout time.Duration // startWatches maintains a list of sources, handlers, and predicates to start when the controller is started. - startWatches []watchDescription + startWatches []source.TypedSource[request] + + // startedEventSourcesAndQueue is used to track if the event sources have been started. + // It ensures that we append sources to c.startWatches only until we call Start() / Warmup() + // It is true if startEventSourcesAndQueueLocked has been called at least once. + startedEventSourcesAndQueue bool + + // didStartEventSourcesOnce is used to ensure that the event sources are only started once. + didStartEventSourcesOnce sync.Once // LogConstructor is used to construct a logger to then log messages to users during reconciliation, // or for example when a watch is started. // Note: LogConstructor has to be able to handle nil requests as we are also using it // outside the context of a reconciliation. - LogConstructor func(request *reconcile.Request) logr.Logger + LogConstructor func(request *request) logr.Logger // RecoverPanic indicates whether the panic caused by reconcile should be recovered. + // Defaults to true. RecoverPanic *bool // LeaderElected indicates whether the controller is leader elected or always running. LeaderElected *bool + + // EnableWarmup specifies whether the controller should start its sources when the manager is not + // the leader. This is useful for cases where sources take a long time to start, as it allows + // for the controller to warm up its caches even before it is elected as the leader. This + // improves leadership failover time, as the caches will be prepopulated before the controller + // transitions to be leader. + // + // Setting EnableWarmup to true and NeedLeaderElection to true means the controller will start its + // sources without waiting to become leader. + // Setting EnableWarmup to true and NeedLeaderElection to false is a no-op as controllers without + // leader election do not wait on leader election to start their sources. + // Defaults to false. + EnableWarmup *bool + + ReconciliationTimeout time.Duration } -// watchDescription contains all the information necessary to start a watch. -type watchDescription struct { - src source.Source - handler handler.EventHandler - predicates []predicate.Predicate +// New returns a new Controller configured with the given options. +func New[request comparable](options Options[request]) *Controller[request] { + return &Controller[request]{ + Do: options.Do, + RateLimiter: options.RateLimiter, + NewQueue: options.NewQueue, + MaxConcurrentReconciles: options.MaxConcurrentReconciles, + CacheSyncTimeout: options.CacheSyncTimeout, + Name: options.Name, + LogConstructor: options.LogConstructor, + RecoverPanic: options.RecoverPanic, + LeaderElected: options.LeaderElected, + EnableWarmup: options.EnableWarmup, + ReconciliationTimeout: options.ReconciliationTimeout, + } } // Reconcile implements reconcile.Reconciler. -func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { +func (c *Controller[request]) Reconcile(ctx context.Context, req request) (_ reconcile.Result, err error) { defer func() { if r := recover(); r != nil { - if c.RecoverPanic != nil && *c.RecoverPanic { + ctrlmetrics.ReconcilePanics.WithLabelValues(c.Name).Inc() + + if c.RecoverPanic == nil || *c.RecoverPanic { for _, fn := range utilruntime.PanicHandlers { - fn(r) + fn(ctx, r) } err = fmt.Errorf("panic: %v [recovered]", r) return @@ -115,36 +206,57 @@ func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (_ re panic(r) } }() + + if c.ReconciliationTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, c.ReconciliationTimeout) + defer cancel() + } + return c.Do.Reconcile(ctx, req) } // Watch implements controller.Controller. -func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prct ...predicate.Predicate) error { +func (c *Controller[request]) Watch(src source.TypedSource[request]) error { c.mu.Lock() defer c.mu.Unlock() - // Controller hasn't started yet, store the watches locally and return. - // - // These watches are going to be held on the controller struct until the manager or user calls Start(...). - if !c.Started { - c.startWatches = append(c.startWatches, watchDescription{src: src, handler: evthdler, predicates: prct}) + // Sources weren't started yet, store the watches locally and return. + // These sources are going to be held until either Warmup() or Start(...) is called. + if !c.startedEventSourcesAndQueue { + c.startWatches = append(c.startWatches, src) return nil } c.LogConstructor(nil).Info("Starting EventSource", "source", src) - return src.Start(c.ctx, evthdler, c.Queue, prct...) + return src.Start(c.ctx, c.Queue) } // NeedLeaderElection implements the manager.LeaderElectionRunnable interface. -func (c *Controller) NeedLeaderElection() bool { +func (c *Controller[request]) NeedLeaderElection() bool { if c.LeaderElected == nil { return true } return *c.LeaderElected } +// Warmup implements the manager.WarmupRunnable interface. +func (c *Controller[request]) Warmup(ctx context.Context) error { + if c.EnableWarmup == nil || !*c.EnableWarmup { + return nil + } + + c.mu.Lock() + defer c.mu.Unlock() + + // Set the ctx so later calls to watch use this internal context + c.ctx = ctx + + return c.startEventSourcesAndQueueLocked(ctx) +} + // Start implements controller.Controller. -func (c *Controller) Start(ctx context.Context) error { +func (c *Controller[request]) Start(ctx context.Context) error { // use an IIFE to get proper lock handling // but lock outside to get proper handling of the queue shutdown c.mu.Lock() @@ -157,64 +269,22 @@ func (c *Controller) Start(ctx context.Context) error { // Set the internal context. c.ctx = ctx - c.Queue = c.MakeQueue() - go func() { - <-ctx.Done() - c.Queue.ShutDown() - }() - wg := &sync.WaitGroup{} err := func() error { defer c.mu.Unlock() // TODO(pwittrock): Reconsider HandleCrash - defer utilruntime.HandleCrash() + defer utilruntime.HandleCrashWithLogger(c.LogConstructor(nil)) // NB(directxman12): launch the sources *before* trying to wait for the - // caches to sync so that they have a chance to register their intendeded + // caches to sync so that they have a chance to register their intended // caches. - for _, watch := range c.startWatches { - c.LogConstructor(nil).Info("Starting EventSource", "source", fmt.Sprintf("%s", watch.src)) - - if err := watch.src.Start(ctx, watch.handler, c.Queue, watch.predicates...); err != nil { - return err - } + if err := c.startEventSourcesAndQueueLocked(ctx); err != nil { + return err } - // Start the SharedIndexInformer factories to begin populating the SharedIndexInformer caches c.LogConstructor(nil).Info("Starting Controller") - for _, watch := range c.startWatches { - syncingSource, ok := watch.src.(source.SyncingSource) - if !ok { - continue - } - - if err := func() error { - // use a context with timeout for launching sources and syncing caches. - sourceStartCtx, cancel := context.WithTimeout(ctx, c.CacheSyncTimeout) - defer cancel() - - // WaitForSync waits for a definitive timeout, and returns if there - // is an error or a timeout - if err := syncingSource.WaitForSync(sourceStartCtx); err != nil { - err := fmt.Errorf("failed to wait for %s caches to sync: %w", c.Name, err) - c.LogConstructor(nil).Error(err, "Could not wait for Cache to sync") - return err - } - - return nil - }(); err != nil { - return err - } - } - - // All the watches have been started, we can reset the local slice. - // - // We should never hold watches more than necessary, each watch source can hold a backing cache, - // which won't be garbage collected if we hold a reference to it. - c.startWatches = nil - // Launch workers to process resources c.LogConstructor(nil).Info("Starting workers", "worker count", c.MaxConcurrentReconciles) wg.Add(c.MaxConcurrentReconciles) @@ -242,10 +312,96 @@ func (c *Controller) Start(ctx context.Context) error { return nil } +// startEventSourcesAndQueueLocked launches all the sources registered with this controller and waits +// for them to sync. It returns an error if any of the sources fail to start or sync. +func (c *Controller[request]) startEventSourcesAndQueueLocked(ctx context.Context) error { + var retErr error + + c.didStartEventSourcesOnce.Do(func() { + queue := c.NewQueue(c.Name, c.RateLimiter) + if priorityQueue, isPriorityQueue := queue.(priorityqueue.PriorityQueue[request]); isPriorityQueue { + c.Queue = priorityQueue + } else { + c.Queue = &priorityQueueWrapper[request]{TypedRateLimitingInterface: queue} + } + go func() { + <-ctx.Done() + c.Queue.ShutDown() + }() + + errGroup := &errgroup.Group{} + for _, watch := range c.startWatches { + log := c.LogConstructor(nil) + _, ok := watch.(interface { + String() string + }) + if !ok { + log = log.WithValues("source", fmt.Sprintf("%T", watch)) + } else { + log = log.WithValues("source", fmt.Sprintf("%s", watch)) + } + didStartSyncingSource := &atomic.Bool{} + errGroup.Go(func() error { + // Use a timeout for starting and syncing the source to avoid silently + // blocking startup indefinitely if it doesn't come up. + sourceStartCtx, cancel := context.WithTimeout(ctx, c.CacheSyncTimeout) + defer cancel() + + sourceStartErrChan := make(chan error, 1) // Buffer chan to not leak goroutine if we time out + go func() { + defer close(sourceStartErrChan) + log.Info("Starting EventSource") + + if err := watch.Start(ctx, c.Queue); err != nil { + sourceStartErrChan <- err + return + } + syncingSource, ok := watch.(source.TypedSyncingSource[request]) + if !ok { + return + } + didStartSyncingSource.Store(true) + if err := syncingSource.WaitForSync(sourceStartCtx); err != nil { + err := fmt.Errorf("failed to wait for %s caches to sync %v: %w", c.Name, syncingSource, err) + log.Error(err, "Could not wait for Cache to sync") + sourceStartErrChan <- err + } + }() + + select { + case err := <-sourceStartErrChan: + return err + case <-sourceStartCtx.Done(): + if didStartSyncingSource.Load() { // We are racing with WaitForSync, wait for it to let it tell us what happened + return <-sourceStartErrChan + } + if ctx.Err() != nil { // Don't return an error if the root context got cancelled + return nil + } + return fmt.Errorf("timed out waiting for source %s to Start. Please ensure that its Start() method is non-blocking", watch) + } + }) + } + retErr = errGroup.Wait() + + // All the watches have been started, we can reset the local slice. + // + // We should never hold watches more than necessary, each watch source can hold a backing cache, + // which won't be garbage collected if we hold a reference to it. + c.startWatches = nil + + // Mark event sources as started after resetting the startWatches slice so that watches from + // a new Watch() call are immediately started. + c.startedEventSourcesAndQueue = true + }) + + return retErr +} + // processNextWorkItem will read a single work item off the workqueue and // attempt to process it, by calling the reconcileHandler. -func (c *Controller) processNextWorkItem(ctx context.Context) bool { - obj, shutdown := c.Queue.Get() +func (c *Controller[request]) processNextWorkItem(ctx context.Context) bool { + obj, priority, shutdown := c.Queue.GetWithPriority() if shutdown { // Stop working return false @@ -262,7 +418,7 @@ func (c *Controller) processNextWorkItem(ctx context.Context) bool { ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(1) defer ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(-1) - c.reconcileHandler(ctx, obj) + c.reconcileHandler(ctx, obj, priority) return true } @@ -273,35 +429,25 @@ const ( labelSuccess = "success" ) -func (c *Controller) initMetrics() { - ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Set(0) - ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Add(0) +func (c *Controller[request]) initMetrics() { ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Add(0) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Add(0) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Add(0) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Add(0) + ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Add(0) + ctrlmetrics.TerminalReconcileErrors.WithLabelValues(c.Name).Add(0) + ctrlmetrics.ReconcilePanics.WithLabelValues(c.Name).Add(0) ctrlmetrics.WorkerCount.WithLabelValues(c.Name).Set(float64(c.MaxConcurrentReconciles)) + ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Set(0) } -func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) { +func (c *Controller[request]) reconcileHandler(ctx context.Context, req request, priority int) { // Update metrics after processing each item reconcileStartTS := time.Now() defer func() { c.updateMetrics(time.Since(reconcileStartTS)) }() - // Make sure that the object is a valid request. - req, ok := obj.(reconcile.Request) - if !ok { - // As the item in the workqueue is actually invalid, we call - // Forget here else we'd go into a loop of attempting to - // process a work item that is invalid. - c.Queue.Forget(obj) - c.LogConstructor(nil).Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj) - // Return true, don't take a break - return - } - log := c.LogConstructor(&req) reconcileID := uuid.NewUUID() @@ -311,43 +457,53 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) { // RunInformersAndControllers the syncHandler, passing it the Namespace/Name string of the // resource to be synced. + log.V(5).Info("Reconciling") result, err := c.Reconcile(ctx, req) + if result.Priority != nil { + priority = *result.Priority + } switch { case err != nil: if errors.Is(err, reconcile.TerminalError(nil)) { ctrlmetrics.TerminalReconcileErrors.WithLabelValues(c.Name).Inc() } else { - c.Queue.AddRateLimited(req) + c.Queue.AddWithOpts(priorityqueue.AddOpts{RateLimited: true, Priority: ptr.To(priority)}, req) } ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc() ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Inc() + if result.RequeueAfter > 0 || result.Requeue { //nolint: staticcheck // We have to handle Requeue until it is removed + log.Info("Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler") + } log.Error(err, "Reconciler error") case result.RequeueAfter > 0: + log.V(5).Info(fmt.Sprintf("Reconcile done, requeueing after %s", result.RequeueAfter)) // The result.RequeueAfter request will be lost, if it is returned // along with a non-nil error. But this is intended as // We need to drive to stable reconcile loops before queuing due // to result.RequestAfter - c.Queue.Forget(obj) - c.Queue.AddAfter(req, result.RequeueAfter) + c.Queue.Forget(req) + c.Queue.AddWithOpts(priorityqueue.AddOpts{After: result.RequeueAfter, Priority: ptr.To(priority)}, req) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Inc() - case result.Requeue: - c.Queue.AddRateLimited(req) + case result.Requeue: //nolint: staticcheck // We have to handle it until it is removed + log.V(5).Info("Reconcile done, requeueing") + c.Queue.AddWithOpts(priorityqueue.AddOpts{RateLimited: true, Priority: ptr.To(priority)}, req) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Inc() default: + log.V(5).Info("Reconcile successful") // Finally, if no error occurs we Forget this item so it does not // get queued again until another change happens. - c.Queue.Forget(obj) + c.Queue.Forget(req) ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Inc() } } // GetLogger returns this controller's logger. -func (c *Controller) GetLogger() logr.Logger { +func (c *Controller[request]) GetLogger() logr.Logger { return c.LogConstructor(nil) } // updateMetrics updates prometheus metrics within the controller. -func (c *Controller) updateMetrics(reconcileTime time.Duration) { +func (c *Controller[request]) updateMetrics(reconcileTime time.Duration) { ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds()) } @@ -368,3 +524,25 @@ type reconcileIDKey struct{} func addReconcileID(ctx context.Context, reconcileID types.UID) context.Context { return context.WithValue(ctx, reconcileIDKey{}, reconcileID) } + +type priorityQueueWrapper[request comparable] struct { + workqueue.TypedRateLimitingInterface[request] +} + +func (p *priorityQueueWrapper[request]) AddWithOpts(opts priorityqueue.AddOpts, items ...request) { + for _, item := range items { + switch { + case opts.RateLimited: + p.TypedRateLimitingInterface.AddRateLimited(item) + case opts.After > 0: + p.TypedRateLimitingInterface.AddAfter(item, opts.After) + default: + p.TypedRateLimitingInterface.Add(item) + } + } +} + +func (p *priorityQueueWrapper[request]) GetWithPriority() (request, int, bool) { + item, shutdown := p.TypedRateLimitingInterface.Get() + return item, 0, shutdown +} diff --git a/pkg/internal/controller/controller_test.go b/pkg/internal/controller/controller_test.go index 05f84dc7b3..6d62b80e22 100644 --- a/pkg/internal/controller/controller_test.go +++ b/pkg/internal/controller/controller_test.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "sync" + "sync/atomic" "time" "github.com/go-logr/logr" @@ -28,28 +29,39 @@ import ( . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" + "go.uber.org/goleak" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" + "sigs.k8s.io/controller-runtime/pkg/controller/priorityqueue" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics" "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/leaderelection" + fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) +type TestRequest struct { + Key string +} + +const testControllerName = "testcontroller" + var _ = Describe("controller", func() { var fakeReconcile *fakeReconciler - var ctrl *Controller + var ctrl *Controller[reconcile.Request] var queue *controllertest.Queue var reconciled chan reconcile.Request var request = reconcile.Request{ @@ -63,23 +75,22 @@ var _ = Describe("controller", func() { results: make(chan fakeReconcileResultPair, 10 /* chosen by the completely scientific approach of guessing */), } queue = &controllertest.Queue{ - Interface: workqueue.New(), + TypedInterface: workqueue.NewTyped[reconcile.Request](), } - ctrl = &Controller{ + ctrl = New[reconcile.Request](Options[reconcile.Request]{ MaxConcurrentReconciles: 1, Do: fakeReconcile, - MakeQueue: func() workqueue.RateLimitingInterface { return queue }, + NewQueue: func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return queue + }, LogConstructor: func(_ *reconcile.Request) logr.Logger { return log.RuntimeLog.WithName("controller").WithName("test") }, - } + }) }) Describe("Reconciler", func() { - It("should call the Reconciler function", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - + It("should call the Reconciler function", func(ctx SpecContext) { ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { return reconcile.Result{Requeue: true}, nil }) @@ -89,13 +100,11 @@ var _ = Describe("controller", func() { Expect(result).To(Equal(reconcile.Result{Requeue: true})) }) - It("should not recover panic if RecoverPanic is false by default", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - + It("should not recover panic if RecoverPanic is false", func(ctx SpecContext) { defer func() { Expect(recover()).ShouldNot(BeNil()) }() + ctrl.RecoverPanic = ptr.To(false) ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { var res *reconcile.Result return *res, nil @@ -104,14 +113,26 @@ var _ = Describe("controller", func() { reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) }) - It("should recover panic if RecoverPanic is true", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should recover panic if RecoverPanic is true by default", func(ctx SpecContext) { + defer func() { + Expect(recover()).To(BeNil()) + }() + // RecoverPanic defaults to true. + ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { + var res *reconcile.Result + return *res, nil + }) + _, err := ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("[recovered]")) + }) + It("should recover panic if RecoverPanic is true", func(ctx SpecContext) { defer func() { Expect(recover()).To(BeNil()) }() - ctrl.RecoverPanic = pointer.Bool(true) + ctrl.RecoverPanic = ptr.To(true) ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { var res *reconcile.Result return *res, nil @@ -121,55 +142,78 @@ var _ = Describe("controller", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("[recovered]")) }) + + It("should time out if ReconciliationTimeout is set", func(ctx SpecContext) { + ctrl.ReconciliationTimeout = time.Duration(1) // One nanosecond + ctrl.Do = reconcile.Func(func(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + <-ctx.Done() + return reconcile.Result{}, ctx.Err() + }) + _, err := ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + Expect(err).To(HaveOccurred()) + Expect(err).To(Equal(context.DeadlineExceeded)) + }) + + It("should not configure a timeout if ReconciliationTimeout is zero", func(ctx SpecContext) { + ctrl.Do = reconcile.Func(func(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + defer GinkgoRecover() + + _, ok := ctx.Deadline() + Expect(ok).To(BeFalse()) + return reconcile.Result{}, nil + }) + _, err := ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + Expect(err).NotTo(HaveOccurred()) + }) }) Describe("Start", func() { - It("should return an error if there is an error waiting for the informers", func() { + It("should return an error if there is an error waiting for the informers", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = time.Second f := false - ctrl.startWatches = []watchDescription{{ - src: source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}), - }} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), + } ctrl.Name = "foo" - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() err := ctrl.Start(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to wait for foo caches to sync")) }) - It("should error when cache sync timeout occurs", func() { - ctrl.CacheSyncTimeout = 10 * time.Nanosecond - + It("should error when cache sync timeout occurs", func(ctx SpecContext) { c, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) c = &cacheWithIndefinitelyBlockingGetInformer{c} - ctrl.startWatches = []watchDescription{{ - src: source.Kind(c, &appsv1.Deployment{}), - }} - ctrl.Name = "testcontroller" + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(c, &appsv1.Deployment{}, &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{}), + } + ctrl.Name = testControllerName - err = ctrl.Start(context.TODO()) + err = ctrl.Start(ctx) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync: timed out waiting for cache to be synced")) + Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync kind source: *v1.Deployment: timed out waiting for cache to be synced")) }) - It("should not error when context cancelled", func() { + It("should not error when controller Start context is cancelled during Sources WaitForSync", func(specCtx SpecContext) { ctrl.CacheSyncTimeout = 1 * time.Second sourceSynced := make(chan struct{}) c, err := cache.New(cfg, cache.Options{}) Expect(err).NotTo(HaveOccurred()) c = &cacheWithIndefinitelyBlockingGetInformer{c} - ctrl.startWatches = []watchDescription{{ - src: &singnallingSourceWrapper{ - SyncingSource: source.Kind(c, &appsv1.Deployment{}), + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), cacheSyncDone: sourceSynced, }, - }} - ctrl.Name = "testcontroller" + } + ctrl.Name = testControllerName - ctx, cancel := context.WithCancel(context.TODO()) + ctx, cancel := context.WithCancel(specCtx) go func() { defer GinkgoRecover() err = ctrl.Start(ctx) @@ -180,26 +224,37 @@ var _ = Describe("controller", func() { <-sourceSynced }) - It("should not error when cache sync timeout is of sufficiently high", func() { - ctrl.CacheSyncTimeout = 1 * time.Second + It("should error when Start() is blocking forever", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = time.Second - ctx, cancel := context.WithCancel(context.Background()) + controllerDone := make(chan struct{}) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-controllerDone + return ctx.Err() + })} + + ctx, cancel := context.WithTimeout(specCtx, 10*time.Second) defer cancel() + err := ctrl.Start(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Please ensure that its Start() method is non-blocking")) + + close(controllerDone) + }) + + It("should not error when cache sync timeout is of sufficiently high", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + sourceSynced := make(chan struct{}) - c, err := cache.New(cfg, cache.Options{}) - Expect(err).NotTo(HaveOccurred()) - ctrl.startWatches = []watchDescription{{ - src: &singnallingSourceWrapper{ - SyncingSource: source.Kind(c, &appsv1.Deployment{}), + c := &informertest.FakeInformers{} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), cacheSyncDone: sourceSynced, }, - }} - - go func() { - defer GinkgoRecover() - Expect(c.Start(ctx)).To(Succeed()) - }() + } go func() { defer GinkgoRecover() @@ -209,15 +264,13 @@ var _ = Describe("controller", func() { <-sourceSynced }) - It("should process events from source.Channel", func() { + It("should process events from source.Channel", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second // channel to be closed when event is processed processed := make(chan struct{}) // source channel ch := make(chan event.GenericEvent, 1) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - // event to be sent to the channel p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, @@ -226,21 +279,20 @@ var _ = Describe("controller", func() { Object: p, } - ins := &source.Channel{Source: ch} - ins.DestBufferSize = 1 - - // send the event to the channel - ch <- evt - - ctrl.startWatches = []watchDescription{{ - src: ins, - handler: handler.Funcs{ - GenericFunc: func(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { + ins := source.Channel( + ch, + handler.Funcs{ + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() close(processed) }, }, - }} + ) + + // send the event to the channel + ch <- evt + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} go func() { defer GinkgoRecover() @@ -249,61 +301,52 @@ var _ = Describe("controller", func() { <-processed }) - It("should error when channel source is not specified", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should error when channel source is not specified", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second - ins := &source.Channel{} - ctrl.startWatches = []watchDescription{{ - src: ins, - }} + ins := source.Channel[string](nil, nil) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} e := ctrl.Start(ctx) Expect(e).To(HaveOccurred()) Expect(e.Error()).To(ContainSubstring("must specify Channel.Source")) }) - It("should call Start on sources with the appropriate EventHandler, Queue, and Predicates", func() { - pr1 := &predicate.Funcs{} - pr2 := &predicate.Funcs{} - evthdl := &handler.EnqueueRequestForObject{} + It("should call Start on sources with the appropriate EventHandler, Queue, and Predicates", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second started := false - src := source.Func(func(ctx context.Context, e handler.EventHandler, q workqueue.RateLimitingInterface, p ...predicate.Predicate) error { + ctx, cancel := context.WithCancel(specCtx) + src := source.Func(func(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { defer GinkgoRecover() - Expect(e).To(Equal(evthdl)) Expect(q).To(Equal(ctrl.Queue)) - Expect(p).To(ConsistOf(pr1, pr2)) started = true + cancel() // Cancel the context so ctrl.Start() doesn't block forever return nil }) - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).NotTo(HaveOccurred()) + Expect(ctrl.Watch(src)).NotTo(HaveOccurred()) - // Use a cancelled context so Start doesn't block - ctx, cancel := context.WithCancel(context.Background()) - cancel() - Expect(ctrl.Start(ctx)).To(Succeed()) + err := ctrl.Start(ctx) + Expect(err).To(Succeed()) Expect(started).To(BeTrue()) }) - It("should return an error if there is an error starting sources", func() { + It("should return an error if there is an error starting sources", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second err := fmt.Errorf("Expected Error: could not start source") - src := source.Func(func(context.Context, handler.EventHandler, - workqueue.RateLimitingInterface, - ...predicate.Predicate) error { + src := source.Func(func(context.Context, + workqueue.TypedRateLimitingInterface[reconcile.Request], + ) error { defer GinkgoRecover() return err }) - Expect(ctrl.Watch(src, &handler.EnqueueRequestForObject{})).To(Succeed()) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + Expect(ctrl.Watch(src)).To(Succeed()) Expect(ctrl.Start(ctx)).To(Equal(err)) }) - It("should return an error if it gets started more than once", func() { + It("should return an error if it gets started more than once", func(specCtx SpecContext) { // Use a cancelled context so Start doesn't block - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(ctrl.Start(ctx)).To(Succeed()) err := ctrl.Start(ctx) @@ -311,56 +354,258 @@ var _ = Describe("controller", func() { Expect(err.Error()).To(Equal("controller was started more than once. This is likely to be caused by being added to a manager multiple times")) }) + It("should check for correct TypedSyncingSource if custom types are used", func(specCtx SpecContext) { + queue := &priorityQueueWrapper[TestRequest]{ + TypedRateLimitingInterface: &controllertest.TypedQueue[TestRequest]{ + TypedInterface: workqueue.NewTyped[TestRequest](), + }} + ctrl := New[TestRequest](Options[TestRequest]{ + NewQueue: func(string, workqueue.TypedRateLimiter[TestRequest]) workqueue.TypedRateLimitingInterface[TestRequest] { + return queue + }, + LogConstructor: func(*TestRequest) logr.Logger { + return log.RuntimeLog.WithName("controller").WithName("test") + }, + }) + ctrl.CacheSyncTimeout = time.Second + src := &bisignallingSource[TestRequest]{ + startCall: make(chan workqueue.TypedRateLimitingInterface[TestRequest]), + startDone: make(chan error, 1), + waitCall: make(chan struct{}), + waitDone: make(chan error, 1), + } + ctrl.startWatches = []source.TypedSource[TestRequest]{src} + ctrl.Name = "foo" + ctx, cancel := context.WithCancel(specCtx) + defer cancel() + startCh := make(chan error) + go func() { + defer GinkgoRecover() + startCh <- ctrl.Start(ctx) + }() + Eventually(src.startCall).Should(Receive(Equal(queue))) + src.startDone <- nil + Eventually(src.waitCall).Should(BeClosed()) + src.waitDone <- nil + cancel() + Eventually(startCh).Should(Receive(Succeed())) + }) }) - Describe("Processing queue items from a Controller", func() { - It("should call Reconciler if an item is enqueued", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + Describe("startEventSourcesAndQueueLocked", func() { + It("should return nil when no sources are provided", func(ctx SpecContext) { + ctrl.startWatches = []source.TypedSource[reconcile.Request]{} + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should initialize controller queue when called", func(ctx SpecContext) { + ctrl.startWatches = []source.TypedSource[reconcile.Request]{} + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctrl.Queue).NotTo(BeNil()) + }) + + It("should return an error if a source fails to start", func(ctx SpecContext) { + expectedErr := fmt.Errorf("failed to start source") + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + // Return the error immediately so we don't get a timeout + return expectedErr + }) + + // Set a sufficiently long timeout to avoid timeouts interfering with the error being returned + ctrl.CacheSyncTimeout = 5 * time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).To(Equal(expectedErr)) + }) + + It("should return an error if a source fails to sync", func(ctx SpecContext) { + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: ptr.To(false)}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), + } + ctrl.Name = "test-controller" + ctrl.CacheSyncTimeout = 5 * time.Second + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to wait for test-controller caches to sync")) + }) + + It("should not return an error when sources start and sync successfully", func(ctx SpecContext) { + // Create a source that starts and syncs successfully + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: ptr.To(true)}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), + } + ctrl.Name = "test-controller" + ctrl.CacheSyncTimeout = 5 * time.Second + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should not return an error when context is cancelled during source sync", func(ctx SpecContext) { + sourceCtx, sourceCancel := context.WithCancel(ctx) + defer sourceCancel() + + ctrl.CacheSyncTimeout = 5 * time.Second + + // Create a bisignallingSource to control the test flow + src := &bisignallingSource[reconcile.Request]{ + startCall: make(chan workqueue.TypedRateLimitingInterface[reconcile.Request]), + startDone: make(chan error, 1), + waitCall: make(chan struct{}), + waitDone: make(chan error, 1), + } + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + + // Start the sources in a goroutine + startErrCh := make(chan error) go func() { defer GinkgoRecover() - Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + startErrCh <- ctrl.startEventSourcesAndQueueLocked(sourceCtx) }() - queue.Add(request) - By("Invoking Reconciler") - fakeReconcile.AddResult(reconcile.Result{}, nil) - Expect(<-reconciled).To(Equal(request)) + // Allow source to start successfully + Eventually(src.startCall).Should(Receive()) + src.startDone <- nil - By("Removing the item from the queue") - Eventually(queue.Len).Should(Equal(0)) - Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) + // Wait for WaitForSync to be called + Eventually(src.waitCall).Should(BeClosed()) + + // Return context.Canceled from WaitForSync + src.waitDone <- context.Canceled + + // Also cancel the context + sourceCancel() + + // We expect to receive the context.Canceled error + err := <-startErrCh + Expect(err).To(MatchError(context.Canceled)) }) - It("should continue to process additional queue items after the first", func() { - ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { + It("should timeout if source Start blocks for too long", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Millisecond + + // Create a source that blocks forever in Start + blockingSrc := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-ctx.Done() + return ctx.Err() + }) + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{blockingSrc} + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("timed out waiting for source")) + }) + + It("should only start sources once when called multiple times concurrently", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Millisecond + + var startCount atomic.Int32 + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + startCount.Add(1) + return nil + }) + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + + By("Calling startEventSourcesAndQueueLocked multiple times in parallel") + var wg sync.WaitGroup + for i := 1; i <= 5; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := ctrl.startEventSourcesAndQueueLocked(ctx) + // All calls should return the same nil error + Expect(err).NotTo(HaveOccurred()) + }() + } + + wg.Wait() + Expect(startCount.Load()).To(Equal(int32(1)), "Source should only be started once even when called multiple times") + }) + + It("should block subsequent calls from returning until the first call to startEventSourcesAndQueueLocked has returned", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 5 * time.Second + + // finishSourceChan is closed to unblock startEventSourcesAndQueueLocked from returning + finishSourceChan := make(chan struct{}) + + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-finishSourceChan + return nil + }) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + + By("Calling startEventSourcesAndQueueLocked asynchronously") + wg := sync.WaitGroup{} + go func() { defer GinkgoRecover() - Fail("Reconciler should not have been called") - return reconcile.Result{}, nil + defer wg.Done() + + wg.Add(1) + Expect(ctrl.startEventSourcesAndQueueLocked(ctx)).To(Succeed()) + }() + + By("Calling startEventSourcesAndQueueLocked again") + var didSubsequentCallComplete atomic.Bool + go func() { + defer GinkgoRecover() + defer wg.Done() + + wg.Add(1) + Expect(ctrl.startEventSourcesAndQueueLocked(ctx)).To(Succeed()) + didSubsequentCallComplete.Store(true) + }() + + // Assert that second call to startEventSourcesAndQueueLocked is blocked while source has not finished + Consistently(didSubsequentCallComplete.Load).Should(BeFalse()) + + By("Finishing source start + sync") + finishSourceChan <- struct{}{} + + // Assert that second call to startEventSourcesAndQueueLocked is now complete + Eventually(didSubsequentCallComplete.Load).Should(BeTrue(), "startEventSourcesAndQueueLocked should complete after source is started and synced") + wg.Wait() + }) + + It("should reset c.startWatches to nil after returning and startedEventSourcesAndQueue", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Millisecond + + src := source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return nil }) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{src} + + err := ctrl.startEventSourcesAndQueueLocked(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(ctrl.startWatches).To(BeNil(), "startWatches should be reset to nil after returning") + Expect(ctrl.startedEventSourcesAndQueue).To(BeTrue(), "startedEventSourcesAndQueue should be set to true after startEventSourcesAndQueueLocked returns without error") + }) + }) + + Describe("Processing queue items from a Controller", func() { + It("should call Reconciler if an item is enqueued", func(ctx SpecContext) { go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) }() + queue.Add(request) - By("adding two bad items to the queue") - queue.Add("foo/bar1") - queue.Add("foo/bar2") + By("Invoking Reconciler") + fakeReconcile.AddResult(reconcile.Result{}, nil) + Expect(<-reconciled).To(Equal(request)) - By("expecting both of them to be skipped") + By("Removing the item from the queue") Eventually(queue.Len).Should(Equal(0)) Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) }) - PIt("should forget an item if it is not a Request and continue processing items", func() { - // TODO(community): write this test - }) - - It("should requeue a Request if there is an error and continue processing items", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should requeue a Request if there is an error and continue processing items", func(ctx SpecContext) { go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -384,9 +629,7 @@ var _ = Describe("controller", func() { Eventually(func() int { return queue.NumRequeues(request) }, 1.0).Should(Equal(0)) }) - It("should not requeue a Request if there is a terminal error", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should not requeue a Request if there is a terminal error", func(ctx SpecContext) { go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -407,12 +650,12 @@ var _ = Describe("controller", func() { // TODO(directxman12): we should ensure that backoff occurrs with error requeue - It("should not reset backoff until there's a non-error result", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should not reset backoff until there's a non-error result", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -443,12 +686,12 @@ var _ = Describe("controller", func() { Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) - It("should requeue a Request with rate limiting if the Result sets Requeue:true and continue processing items", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should requeue a Request with rate limiting if the Result sets Requeue:true and continue processing items", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -473,12 +716,70 @@ var _ = Describe("controller", func() { Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) - It("should requeue a Request after a duration (but not rate-limitted) if the Result sets RequeueAfter (regardless of Requeue)", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should retain the priority when the reconciler requests a requeue", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will request a requeue") + fakeReconcile.AddResult(reconcile.Result{Requeue: true}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(10), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should use the priority from Result when the reconciler requests a requeue", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will request a requeue") + fakeReconcile.AddResult(reconcile.Result{Requeue: true, Priority: ptr.To(99)}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(99), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should requeue a Request after a duration (but not rate-limited) if the Result sets RequeueAfter (regardless of Requeue)", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -503,12 +804,70 @@ var _ = Describe("controller", func() { Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) - It("should perform error behavior if error is not nil, regardless of RequeueAfter", func() { - dq := &DelegatingQueue{RateLimitingInterface: ctrl.MakeQueue()} - ctrl.MakeQueue = func() workqueue.RateLimitingInterface { return dq } + It("should retain the priority with RequeueAfter", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will ask for RequeueAfter") + fakeReconcile.AddResult(reconcile.Result{RequeueAfter: time.Millisecond * 100}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + After: time.Millisecond * 100, + Priority: ptr.To(10), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should use the priority from Result with RequeueAfter", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will ask for RequeueAfter") + fakeReconcile.AddResult(reconcile.Result{RequeueAfter: time.Millisecond * 100, Priority: ptr.To(99)}, nil) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + After: time.Millisecond * 100, + Priority: ptr.To(99), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should perform error behavior if error is not nil, regardless of RequeueAfter", func(ctx SpecContext) { + dq := &DelegatingQueue{TypedRateLimitingInterface: ctrl.NewQueue("controller1", nil)} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return dq + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -532,16 +891,74 @@ var _ = Describe("controller", func() { Eventually(func() int { return dq.NumRequeues(request) }).Should(Equal(0)) }) - PIt("should return if the queue is shutdown", func() { - // TODO(community): write this test - }) + It("should retain the priority when there was an error", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } - PIt("should wait for informers to be synced before processing items", func() { - // TODO(community): write this test - }) + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() - PIt("should create a new go routine for MaxConcurrentReconciles", func() { - // TODO(community): write this test + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will return an error") + fakeReconcile.AddResult(reconcile.Result{}, errors.New("oups, I did it again")) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(10), + }, + items: []reconcile.Request{request}, + }})) + }) + + It("should use the priority from Result when there was an error", func(ctx SpecContext) { + q := &fakePriorityQueue{PriorityQueue: priorityqueue.New[reconcile.Request]("controller1")} + ctrl.NewQueue = func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return q + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) + }() + + q.PriorityQueue.AddWithOpts(priorityqueue.AddOpts{Priority: ptr.To(10)}, request) + + By("Invoking Reconciler which will return an error") + fakeReconcile.AddResult(reconcile.Result{Priority: ptr.To(99)}, errors.New("oups, I did it again")) + Expect(<-reconciled).To(Equal(request)) + Eventually(func() []priorityQueueAddition { + q.lock.Lock() + defer q.lock.Unlock() + return q.added + }).Should(Equal([]priorityQueueAddition{{ + AddOpts: priorityqueue.AddOpts{ + RateLimited: true, + Priority: ptr.To(99), + }, + items: []reconcile.Request{request}, + }})) + }) + + PIt("should return if the queue is shutdown", func() { + // TODO(community): write this test + }) + + PIt("should wait for informers to be synced before processing items", func() { + // TODO(community): write this test + }) + + PIt("should create a new go routine for MaxConcurrentReconciles", func() { + // TODO(community): write this test }) Context("prometheus metric reconcile_total", func() { @@ -552,7 +969,7 @@ var _ = Describe("controller", func() { reconcileTotal.Reset() }) - It("should get updated on successful reconciliation", func() { + It("should get updated on successful reconciliation", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "success").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -561,8 +978,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -581,7 +996,7 @@ var _ = Describe("controller", func() { }, 2.0).Should(Succeed()) }) - It("should get updated on reconcile errors", func() { + It("should get updated on reconcile errors", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "error").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -590,8 +1005,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -610,7 +1023,7 @@ var _ = Describe("controller", func() { }, 2.0).Should(Succeed()) }) - It("should get updated when reconcile returns with retry enabled", func() { + It("should get updated when reconcile returns with retry enabled", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -619,8 +1032,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -640,7 +1051,7 @@ var _ = Describe("controller", func() { }, 2.0).Should(Succeed()) }) - It("should get updated when reconcile returns with retryAfter enabled", func() { + It("should get updated when reconcile returns with retryAfter enabled", func(ctx SpecContext) { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry_after").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -649,8 +1060,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -671,7 +1080,7 @@ var _ = Describe("controller", func() { }) Context("should update prometheus metrics", func() { - It("should requeue a Request if there is an error and continue processing items", func() { + It("should requeue a Request if there is an error and continue processing items", func(ctx SpecContext) { var reconcileErrs dto.Metric ctrlmetrics.ReconcileErrors.Reset() Expect(func() error { @@ -682,8 +1091,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -710,7 +1117,7 @@ var _ = Describe("controller", func() { Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) }) - It("should add a reconcile time to the reconcile time histogram", func() { + It("should add a reconcile time to the reconcile time histogram", func(ctx SpecContext) { var reconcileTime dto.Metric ctrlmetrics.ReconcileTime.Reset() @@ -724,8 +1131,6 @@ var _ = Describe("controller", func() { return nil }()).Should(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(ctrl.Start(ctx)).NotTo(HaveOccurred()) @@ -752,19 +1157,527 @@ var _ = Describe("controller", func() { }) }) }) + + Describe("Warmup", func() { + JustBeforeEach(func() { + ctrl.EnableWarmup = ptr.To(true) + }) + + It("should track warmup status correctly with successful sync", func(ctx SpecContext) { + // Setup controller with sources that complete successfully + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return nil + }), + } + + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }) + + It("should return an error if there is an error waiting for the informers", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(&informertest.FakeInformers{Synced: ptr.To(false)}, &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{}), + } + ctrl.Name = testControllerName + err := ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync")) + }) + + It("should error when cache sync timeout occurs", func(ctx SpecContext) { + c, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + c = &cacheWithIndefinitelyBlockingGetInformer{c} + + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Kind(c, &appsv1.Deployment{}, &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{}), + } + ctrl.Name = testControllerName + + err = ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync kind source: *v1.Deployment: timed out waiting for cache to be synced")) + }) + + It("should not error when controller Warmup context is cancelled during Sources WaitForSync", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 1 * time.Second + + sourceSynced := make(chan struct{}) + c, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + c = &cacheWithIndefinitelyBlockingGetInformer{c} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), + cacheSyncDone: sourceSynced, + }, + } + ctrl.Name = testControllerName + + ctx, cancel := context.WithCancel(specCtx) + go func() { + defer GinkgoRecover() + err = ctrl.Warmup(ctx) + Expect(err).To(Succeed()) + }() + + cancel() + <-sourceSynced + }) + + It("should error when Warmup() is blocking forever", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = time.Second + + controllerDone := make(chan struct{}) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-controllerDone + return ctx.Err() + })} + + ctx, cancel := context.WithTimeout(specCtx, 10*time.Second) + defer cancel() + + err := ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Please ensure that its Start() method is non-blocking")) + + close(controllerDone) + }) + + It("should not error when cache sync timeout is of sufficiently high", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + + sourceSynced := make(chan struct{}) + c := &informertest.FakeInformers{} + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + &singnallingSourceWrapper{ + SyncingSource: source.Kind[client.Object](c, &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}), + cacheSyncDone: sourceSynced, + }, + } + + go func() { + defer GinkgoRecover() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + + <-sourceSynced + }) + + It("should process events from source.Channel", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + // channel to be closed when event is processed + processed := make(chan struct{}) + // source channel + ch := make(chan event.GenericEvent, 1) + + // event to be sent to the channel + p := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, + } + evt := event.GenericEvent{ + Object: p, + } + + ins := source.Channel( + ch, + handler.Funcs{ + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + close(processed) + }, + }, + ) + + // send the event to the channel + ch <- evt + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} + + go func() { + defer GinkgoRecover() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + <-processed + }) + + It("should error when channel source is not specified", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + + ins := source.Channel[string](nil, nil) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ins} + + e := ctrl.Warmup(ctx) + Expect(e).To(HaveOccurred()) + Expect(e.Error()).To(ContainSubstring("must specify Channel.Source")) + }) + + It("should call Start on sources with the appropriate EventHandler, Queue, and Predicates", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + started := false + ctx, cancel := context.WithCancel(specCtx) + src := source.Func(func(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + defer GinkgoRecover() + Expect(q).To(Equal(ctrl.Queue)) + + started = true + cancel() // Cancel the context so ctrl.Warmup() doesn't block forever + return nil + }) + Expect(ctrl.Watch(src)).NotTo(HaveOccurred()) + + err := ctrl.Warmup(ctx) + Expect(err).To(Succeed()) + Expect(started).To(BeTrue()) + }) + + It("should return an error if there is an error starting sources", func(ctx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + err := fmt.Errorf("Expected Error: could not start source") + src := source.Func(func(context.Context, + workqueue.TypedRateLimitingInterface[reconcile.Request], + ) error { + defer GinkgoRecover() + return err + }) + Expect(ctrl.Watch(src)).To(Succeed()) + + Expect(ctrl.Warmup(ctx)).To(Equal(err)) + }) + + It("should track warmup status correctly with unsuccessful sync", func(ctx SpecContext) { + // Setup controller with sources that complete with error + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return errors.New("sync error") + }), + } + + err := ctrl.Warmup(ctx) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("sync error")) + }) + + It("should call Start on sources with the appropriate non-nil queue", func(specCtx SpecContext) { + ctrl.CacheSyncTimeout = 10 * time.Second + started := false + ctx, cancel := context.WithCancel(specCtx) + src := source.Func(func(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + defer GinkgoRecover() + Expect(q).ToNot(BeNil()) + Expect(q).To(Equal(ctrl.Queue)) + + started = true + cancel() // Cancel the context so ctrl.Start() doesn't block forever + return nil + }) + Expect(ctrl.Watch(src)).To(Succeed()) + Expect(ctrl.Warmup(ctx)).To(Succeed()) + Expect(ctrl.Queue).ToNot(BeNil()) + Expect(started).To(BeTrue()) + }) + + It("should return true if context is cancelled while waiting for source to start", func(specCtx SpecContext) { + // Setup controller with sources that complete with error + ctx, cancel := context.WithCancel(specCtx) + defer cancel() + + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-ctx.Done() + return nil + }), + } + + // channel to prevent the goroutine from outliving the It test + waitChan := make(chan struct{}) + + // Invoked in a goroutine because Warmup will block + go func() { + defer GinkgoRecover() + defer close(waitChan) + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + + cancel() + <-waitChan + }) + + It("should be called before leader election runnables if warmup is enabled", func(specCtx SpecContext) { + // This unit test exists to ensure that a warmup enabled controller will actually be + // called in the warmup phase before the leader election runnables are started. It + // catches regressions in the controller that would not implement warmupRunnable from + // pkg/manager. + ctx, cancel := context.WithCancel(specCtx) + + By("Creating a channel to track execution order") + runnableExecutionOrderChan := make(chan string, 2) + const nonWarmupRunnableName = "nonWarmupRunnable" + const warmupRunnableName = "warmupRunnable" + + ctrl.CacheSyncTimeout = time.Second + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + runnableExecutionOrderChan <- warmupRunnableName + return nil + }), + } + + nonWarmupCtrl := New[reconcile.Request](Options[reconcile.Request]{ + MaxConcurrentReconciles: 1, + Do: fakeReconcile, + NewQueue: func(string, workqueue.TypedRateLimiter[reconcile.Request]) workqueue.TypedRateLimitingInterface[reconcile.Request] { + return queue + }, + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return log.RuntimeLog.WithName("controller").WithName("test") + }, + CacheSyncTimeout: time.Second, + EnableWarmup: ptr.To(false), + LeaderElected: ptr.To(true), + }) + nonWarmupCtrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + runnableExecutionOrderChan <- nonWarmupRunnableName + return nil + }), + } + + By("Creating a test resource lock with hooks") + resourceLock, err := fakeleaderelection.NewResourceLock(nil, nil, leaderelection.Options{}) + Expect(err).ToNot(HaveOccurred()) + + By("Creating a manager") + testenv = &envtest.Environment{} + cfg, err := testenv.Start() + Expect(err).NotTo(HaveOccurred()) + m, err := manager.New(cfg, manager.Options{ + LeaderElection: true, + LeaderElectionID: "some-leader-election-id", + LeaderElectionNamespace: "default", + LeaderElectionResourceLockInterface: resourceLock, + }) + Expect(err).NotTo(HaveOccurred()) + + By("Adding warmup and non-warmup controllers to the manager") + Expect(m.Add(ctrl)).To(Succeed()) + Expect(m.Add(nonWarmupCtrl)).To(Succeed()) + + By("Blocking leader election") + resourceLockWithHooks, ok := resourceLock.(fakeleaderelection.ControllableResourceLockInterface) + Expect(ok).To(BeTrue(), "resource lock should implement ResourceLockInterfaceWithHooks") + resourceLockWithHooks.BlockLeaderElection() + + By("Starting the manager") + waitChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(waitChan) + Expect(m.Start(ctx)).To(Succeed()) + }() + Expect(<-runnableExecutionOrderChan).To(Equal(warmupRunnableName)) + + By("Unblocking leader election") + resourceLockWithHooks.UnblockLeaderElection() + <-m.Elected() + Expect(<-runnableExecutionOrderChan).To(Equal(nonWarmupRunnableName)) + + cancel() + <-waitChan + }) + + It("should not cause a data race when called concurrently", func(ctx SpecContext) { + + ctrl.CacheSyncTimeout = time.Second + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + return nil + }), + } + + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go func() { + defer GinkgoRecover() + defer wg.Done() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + }() + } + + wg.Wait() + }) + + It("should not cause a data race when called concurrently with Start and only start sources once", func(specCtx SpecContext) { + ctx, cancel := context.WithCancel(specCtx) + + ctrl.CacheSyncTimeout = time.Second + numWatches := 10 + + var watchStartedCount atomic.Int32 + for range numWatches { + ctrl.startWatches = append(ctrl.startWatches, source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + watchStartedCount.Add(1) + return nil + })) + } + + By("calling Warmup and Start concurrently") + blockOnStartChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).To(Succeed()) + close(blockOnStartChan) + }() + + blockOnWarmupChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(ctrl.Warmup(ctx)).To(Succeed()) + close(blockOnWarmupChan) + }() + + <-blockOnWarmupChan + + cancel() + + <-blockOnStartChan + + Expect(watchStartedCount.Load()).To(Equal(int32(numWatches)), "source should only be started once") + Expect(ctrl.startWatches).To(BeNil(), "startWatches should be reset to nil after they are started") + }) + + It("should start sources added after Warmup is called", func(specCtx SpecContext) { + ctx, cancel := context.WithCancel(specCtx) + + ctrl.CacheSyncTimeout = time.Second + + Expect(ctrl.Warmup(ctx)).To(Succeed()) + + By("starting a watch after warmup is added") + var didWatchStart atomic.Bool + Expect(ctrl.Watch(source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + didWatchStart.Store(true) + return nil + }))).To(Succeed()) + + waitChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).To(Succeed()) + close(waitChan) + }() + + Eventually(didWatchStart.Load).Should(BeTrue(), "watch should be started if it is added after Warmup") + + cancel() + <-waitChan + }) + + DescribeTable("should not leak goroutines when manager is stopped with warmup runnable", + func(specContext SpecContext, leaderElection bool) { + ctx, cancel := context.WithCancel(specContext) + defer cancel() + + ctrl.CacheSyncTimeout = time.Second + + By("Creating a manager") + testenv = &envtest.Environment{} + cfg, err := testenv.Start() + Expect(err).NotTo(HaveOccurred()) + m, err := manager.New(cfg, manager.Options{ + LeaderElection: leaderElection, + LeaderElectionID: "some-leader-election-id", + LeaderElectionNamespace: "default", + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + <-ctx.Done() + return nil + }), + } + Expect(m.Add(ctrl)).To(Succeed()) + + // ignore needs to go after the testenv.Start() call to ignore the apiserver + // process + currentGRs := goleak.IgnoreCurrent() + waitChan := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).To(Succeed()) + close(waitChan) + }() + + <-m.Elected() + By("stopping the manager via context") + cancel() + + Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) + <-waitChan + }, + Entry("and with leader election enabled", true), + Entry("and without leader election enabled", false), + ) + }) + + Describe("Warmup with warmup disabled", func() { + JustBeforeEach(func() { + ctrl.EnableWarmup = ptr.To(false) + }) + + It("should not start sources when Warmup is called if warmup is disabled but start it when Start is called.", func(specCtx SpecContext) { + // Setup controller with sources that complete successfully + ctx, cancel := context.WithCancel(specCtx) + + ctrl.CacheSyncTimeout = time.Second + var isSourceStarted atomic.Bool + isSourceStarted.Store(false) + ctrl.startWatches = []source.TypedSource[reconcile.Request]{ + source.Func(func(ctx context.Context, _ workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + isSourceStarted.Store(true) + return nil + }), + } + + By("Calling Warmup when EnableWarmup is false") + err := ctrl.Warmup(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(isSourceStarted.Load()).To(BeFalse()) + + By("Calling Start when EnableWarmup is false") + waitChan := make(chan struct{}) + + go func() { + defer GinkgoRecover() + Expect(ctrl.Start(ctx)).To(Succeed()) + close(waitChan) + }() + Eventually(isSourceStarted.Load).Should(BeTrue()) + cancel() + <-waitChan + }) + }) }) var _ = Describe("ReconcileIDFromContext function", func() { - It("should return an empty string if there is nothing in the context", func() { - ctx := context.Background() + It("should return an empty string if there is nothing in the context", func(ctx SpecContext) { reconcileID := ReconcileIDFromContext(ctx) Expect(reconcileID).To(Equal(types.UID(""))) }) - It("should return the correct reconcileID from context", func() { + It("should return the correct reconcileID from context", func(specContext SpecContext) { const expectedReconcileID = types.UID("uuid") - ctx := addReconcileID(context.Background(), expectedReconcileID) + ctx := addReconcileID(specContext, expectedReconcileID) reconcileID := ReconcileIDFromContext(ctx) Expect(reconcileID).To(Equal(expectedReconcileID)) @@ -772,7 +1685,7 @@ var _ = Describe("ReconcileIDFromContext function", func() { }) type DelegatingQueue struct { - workqueue.RateLimitingInterface + workqueue.TypedRateLimitingInterface[reconcile.Request] mu sync.Mutex countAddRateLimited int @@ -780,36 +1693,36 @@ type DelegatingQueue struct { countAddAfter int } -func (q *DelegatingQueue) AddRateLimited(item interface{}) { +func (q *DelegatingQueue) AddRateLimited(item reconcile.Request) { q.mu.Lock() defer q.mu.Unlock() q.countAddRateLimited++ - q.RateLimitingInterface.AddRateLimited(item) + q.TypedRateLimitingInterface.AddRateLimited(item) } -func (q *DelegatingQueue) AddAfter(item interface{}, d time.Duration) { +func (q *DelegatingQueue) AddAfter(item reconcile.Request, d time.Duration) { q.mu.Lock() defer q.mu.Unlock() q.countAddAfter++ - q.RateLimitingInterface.AddAfter(item, d) + q.TypedRateLimitingInterface.AddAfter(item, d) } -func (q *DelegatingQueue) Add(item interface{}) { +func (q *DelegatingQueue) Add(item reconcile.Request) { q.mu.Lock() defer q.mu.Unlock() q.countAdd++ - q.RateLimitingInterface.Add(item) + q.TypedRateLimitingInterface.Add(item) } -func (q *DelegatingQueue) Forget(item interface{}) { +func (q *DelegatingQueue) Forget(item reconcile.Request) { q.mu.Lock() defer q.mu.Unlock() q.countAdd-- - q.RateLimitingInterface.Forget(item) + q.TypedRateLimitingInterface.Forget(item) } type countInfo struct { @@ -854,6 +1767,15 @@ type singnallingSourceWrapper struct { source.SyncingSource } +func (s *singnallingSourceWrapper) Start(ctx context.Context, q workqueue.TypedRateLimitingInterface[reconcile.Request]) error { + err := s.SyncingSource.Start(ctx, q) + if err != nil { + // WaitForSync will never be called if this errors, so close the channel to prevent deadlocks in tests + close(s.cacheSyncDone) + } + return err +} + func (s *singnallingSourceWrapper) WaitForSync(ctx context.Context) error { defer func() { close(s.cacheSyncDone) @@ -873,7 +1795,62 @@ type cacheWithIndefinitelyBlockingGetInformer struct { cache.Cache } -func (c *cacheWithIndefinitelyBlockingGetInformer) GetInformer(ctx context.Context, obj client.Object) (cache.Informer, error) { +func (c *cacheWithIndefinitelyBlockingGetInformer) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { <-ctx.Done() return nil, errors.New("GetInformer timed out") } + +type bisignallingSource[T comparable] struct { + // receives the queue that is passed to Start + startCall chan workqueue.TypedRateLimitingInterface[T] + // passes an error to return from Start + startDone chan error + // closed when WaitForSync is called + waitCall chan struct{} + // passes an error to return from WaitForSync + waitDone chan error +} + +var _ source.TypedSyncingSource[int] = (*bisignallingSource[int])(nil) + +func (t *bisignallingSource[T]) Start(ctx context.Context, q workqueue.TypedRateLimitingInterface[T]) error { + select { + case t.startCall <- q: + case <-ctx.Done(): + return ctx.Err() + } + select { + case err := <-t.startDone: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (t *bisignallingSource[T]) WaitForSync(ctx context.Context) error { + close(t.waitCall) + select { + case err := <-t.waitDone: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +type priorityQueueAddition struct { + priorityqueue.AddOpts + items []reconcile.Request +} + +type fakePriorityQueue struct { + priorityqueue.PriorityQueue[reconcile.Request] + + lock sync.Mutex + added []priorityQueueAddition +} + +func (f *fakePriorityQueue) AddWithOpts(o priorityqueue.AddOpts, items ...reconcile.Request) { + f.lock.Lock() + defer f.lock.Unlock() + f.added = append(f.added, priorityQueueAddition{AddOpts: o, items: items}) +} diff --git a/pkg/internal/controller/metrics/metrics.go b/pkg/internal/controller/metrics/metrics.go index b74ce062be..450e9ae25b 100644 --- a/pkg/internal/controller/metrics/metrics.go +++ b/pkg/internal/controller/metrics/metrics.go @@ -17,6 +17,8 @@ limitations under the License. package metrics import ( + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -46,6 +48,13 @@ var ( Help: "Total number of terminal reconciliation errors per controller", }, []string{"controller"}) + // ReconcilePanics is a prometheus counter metrics which holds the total + // number of panics from the Reconciler. + ReconcilePanics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "controller_runtime_reconcile_panics_total", + Help: "Total number of reconciliation panics per controller", + }, []string{"controller"}) + // ReconcileTime is a prometheus metric which keeps track of the duration // of reconciliations. ReconcileTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ @@ -53,6 +62,9 @@ var ( Help: "Length of time per reconciliation per controller", Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60}, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"controller"}) // WorkerCount is a prometheus metric which holds the number of @@ -75,12 +87,13 @@ func init() { ReconcileTotal, ReconcileErrors, TerminalReconcileErrors, + ReconcilePanics, ReconcileTime, WorkerCount, ActiveWorkers, // expose process metrics like CPU, Memory, file descriptor usage etc. collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), - // expose Go runtime metrics like GC stats, memory stats etc. - collectors.NewGoCollector(), + // expose all Go runtime metrics like GC stats, memory stats etc. + collectors.NewGoCollector(collectors.WithGoCollectorRuntimeMetrics(collectors.MetricsAll)), ) } diff --git a/pkg/internal/field/selector/utils.go b/pkg/internal/field/selector/utils.go index 4f6d084318..8f6dc71ede 100644 --- a/pkg/internal/field/selector/utils.go +++ b/pkg/internal/field/selector/utils.go @@ -22,14 +22,16 @@ import ( ) // RequiresExactMatch checks if the given field selector is of the form `k=v` or `k==v`. -func RequiresExactMatch(sel fields.Selector) (field, val string, required bool) { +func RequiresExactMatch(sel fields.Selector) bool { reqs := sel.Requirements() - if len(reqs) != 1 { - return "", "", false + if len(reqs) == 0 { + return false } - req := reqs[0] - if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { - return "", "", false + + for _, req := range reqs { + if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { + return false + } } - return req.Field, req.Value, true + return true } diff --git a/pkg/internal/field/selector/utils_test.go b/pkg/internal/field/selector/utils_test.go index fba214ff16..a48bbf4e5a 100644 --- a/pkg/internal/field/selector/utils_test.go +++ b/pkg/internal/field/selector/utils_test.go @@ -27,62 +27,32 @@ import ( var _ = Describe("RequiresExactMatch function", func() { It("Returns false when the selector matches everything", func() { - _, _, requiresExactMatch := RequiresExactMatch(fields.Everything()) + requiresExactMatch := RequiresExactMatch(fields.Everything()) Expect(requiresExactMatch).To(BeFalse()) }) It("Returns false when the selector matches nothing", func() { - _, _, requiresExactMatch := RequiresExactMatch(fields.Nothing()) + requiresExactMatch := RequiresExactMatch(fields.Nothing()) Expect(requiresExactMatch).To(BeFalse()) }) It("Returns false when the selector has the form key!=val", func() { - _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key!=val")) + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key!=val")) Expect(requiresExactMatch).To(BeFalse()) }) - It("Returns false when the selector has the form key1==val1,key2==val2", func() { - _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key1==val1,key2==val2")) - Expect(requiresExactMatch).To(BeFalse()) + It("Returns true when the selector has the form key1==val1,key2==val2", func() { + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key1==val1,key2==val2")) + Expect(requiresExactMatch).To(BeTrue()) }) It("Returns true when the selector has the form key==val", func() { - _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key==val")) + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key==val")) Expect(requiresExactMatch).To(BeTrue()) }) It("Returns true when the selector has the form key=val", func() { - _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key=val")) + requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key=val")) Expect(requiresExactMatch).To(BeTrue()) }) - - It("Returns empty key and value when the selector matches everything", func() { - key, val, _ := RequiresExactMatch(fields.Everything()) - Expect(key).To(Equal("")) - Expect(val).To(Equal("")) - }) - - It("Returns empty key and value when the selector matches nothing", func() { - key, val, _ := RequiresExactMatch(fields.Nothing()) - Expect(key).To(Equal("")) - Expect(val).To(Equal("")) - }) - - It("Returns empty key and value when the selector has the form key!=val", func() { - key, val, _ := RequiresExactMatch(fields.ParseSelectorOrDie("key!=val")) - Expect(key).To(Equal("")) - Expect(val).To(Equal("")) - }) - - It("Returns key and value when the selector has the form key==val", func() { - key, val, _ := RequiresExactMatch(fields.ParseSelectorOrDie("key==val")) - Expect(key).To(Equal("key")) - Expect(val).To(Equal("val")) - }) - - It("Returns key and value when the selector has the form key=val", func() { - key, val, _ := RequiresExactMatch(fields.ParseSelectorOrDie("key=val")) - Expect(key).To(Equal("key")) - Expect(val).To(Equal("val")) - }) }) diff --git a/pkg/internal/metrics/workqueue.go b/pkg/internal/metrics/workqueue.go new file mode 100644 index 0000000000..402319817b --- /dev/null +++ b/pkg/internal/metrics/workqueue.go @@ -0,0 +1,170 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// This file is copied and adapted from k8s.io/component-base/metrics/prometheus/workqueue +// which registers metrics to the k8s legacy Registry. We require very +// similar functionality, but must register metrics to a different Registry. + +// Metrics subsystem and all keys used by the workqueue. +const ( + WorkQueueSubsystem = metrics.WorkQueueSubsystem + DepthKey = metrics.DepthKey + AddsKey = metrics.AddsKey + QueueLatencyKey = metrics.QueueLatencyKey + WorkDurationKey = metrics.WorkDurationKey + UnfinishedWorkKey = metrics.UnfinishedWorkKey + LongestRunningProcessorKey = metrics.LongestRunningProcessorKey + RetriesKey = metrics.RetriesKey +) + +var ( + depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Subsystem: WorkQueueSubsystem, + Name: DepthKey, + Help: "Current depth of workqueue by workqueue and priority", + }, []string{"name", "controller", "priority"}) + + adds = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: WorkQueueSubsystem, + Name: AddsKey, + Help: "Total number of adds handled by workqueue", + }, []string{"name", "controller"}) + + latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Subsystem: WorkQueueSubsystem, + Name: QueueLatencyKey, + Help: "How long in seconds an item stays in workqueue before being requested", + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 12), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"name", "controller"}) + + workDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Subsystem: WorkQueueSubsystem, + Name: WorkDurationKey, + Help: "How long in seconds processing an item from workqueue takes.", + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 12), + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"name", "controller"}) + + unfinished = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Subsystem: WorkQueueSubsystem, + Name: UnfinishedWorkKey, + Help: "How many seconds of work has been done that " + + "is in progress and hasn't been observed by work_duration. Large " + + "values indicate stuck threads. One can deduce the number of stuck " + + "threads by observing the rate at which this increases.", + }, []string{"name", "controller"}) + + longestRunningProcessor = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Subsystem: WorkQueueSubsystem, + Name: LongestRunningProcessorKey, + Help: "How many seconds has the longest running " + + "processor for workqueue been running.", + }, []string{"name", "controller"}) + + retries = prometheus.NewCounterVec(prometheus.CounterOpts{ + Subsystem: WorkQueueSubsystem, + Name: RetriesKey, + Help: "Total number of retries handled by workqueue", + }, []string{"name", "controller"}) +) + +func init() { + metrics.Registry.MustRegister(depth) + metrics.Registry.MustRegister(adds) + metrics.Registry.MustRegister(latency) + metrics.Registry.MustRegister(workDuration) + metrics.Registry.MustRegister(unfinished) + metrics.Registry.MustRegister(longestRunningProcessor) + metrics.Registry.MustRegister(retries) + + workqueue.SetProvider(WorkqueueMetricsProvider{}) +} + +type WorkqueueMetricsProvider struct{} + +func (WorkqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { + return depth.WithLabelValues(name, name, "") // no priority +} + +func (WorkqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { + return adds.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { + return latency.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { + return workDuration.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { + return unfinished.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { + return longestRunningProcessor.WithLabelValues(name, name) +} + +func (WorkqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { + return retries.WithLabelValues(name, name) +} + +type MetricsProviderWithPriority interface { + workqueue.MetricsProvider + + NewDepthMetricWithPriority(name string) DepthMetricWithPriority +} + +// DepthMetricWithPriority represents a depth metric with priority. +type DepthMetricWithPriority interface { + Inc(priority int) + Dec(priority int) +} + +var _ MetricsProviderWithPriority = WorkqueueMetricsProvider{} + +func (WorkqueueMetricsProvider) NewDepthMetricWithPriority(name string) DepthMetricWithPriority { + return &depthWithPriorityMetric{lvs: []string{name, name}} +} + +type depthWithPriorityMetric struct { + lvs []string +} + +func (g *depthWithPriorityMetric) Inc(priority int) { + depth.WithLabelValues(append(g.lvs, strconv.Itoa(priority))...).Inc() +} + +func (g *depthWithPriorityMetric) Dec(priority int) { + depth.WithLabelValues(append(g.lvs, strconv.Itoa(priority))...).Dec() +} diff --git a/pkg/internal/recorder/recorder_integration_test.go b/pkg/internal/recorder/recorder_integration_test.go index 130a306053..c278fbde79 100644 --- a/pkg/internal/recorder/recorder_integration_test.go +++ b/pkg/internal/recorder/recorder_integration_test.go @@ -37,7 +37,7 @@ import ( var _ = Describe("recorder", func() { Describe("recorder", func() { - It("should publish events", func() { + It("should publish events", func(ctx SpecContext) { By("Creating the Manager") cm, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -56,12 +56,10 @@ var _ = Describe("recorder", func() { Expect(err).NotTo(HaveOccurred()) By("Watching Resources") - err = instance.Watch(source.Kind(cm.GetCache(), &appsv1.Deployment{}), &handler.EnqueueRequestForObject{}) + err = instance.Watch(source.Kind(cm.GetCache(), &appsv1.Deployment{}, &handler.TypedEnqueueRequestForObject[*appsv1.Deployment]{})) Expect(err).NotTo(HaveOccurred()) By("Starting the Manager") - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(cm.Start(ctx)).NotTo(HaveOccurred()) diff --git a/pkg/internal/source/event_handler.go b/pkg/internal/source/event_handler.go index ae8404a1fa..7cc8c51555 100644 --- a/pkg/internal/source/event_handler.go +++ b/pkg/internal/source/event_handler.go @@ -32,9 +32,15 @@ import ( var log = logf.RuntimeLog.WithName("source").WithName("EventHandler") +var _ cache.ResourceEventHandler = &EventHandler[client.Object, any]{} + // NewEventHandler creates a new EventHandler. -func NewEventHandler(ctx context.Context, queue workqueue.RateLimitingInterface, handler handler.EventHandler, predicates []predicate.Predicate) *EventHandler { - return &EventHandler{ +func NewEventHandler[object client.Object, request comparable]( + ctx context.Context, + queue workqueue.TypedRateLimitingInterface[request], + handler handler.TypedEventHandler[object, request], + predicates []predicate.TypedPredicate[object]) *EventHandler[object, request] { + return &EventHandler[object, request]{ ctx: ctx, handler: handler, queue: queue, @@ -43,32 +49,24 @@ func NewEventHandler(ctx context.Context, queue workqueue.RateLimitingInterface, } // EventHandler adapts a handler.EventHandler interface to a cache.ResourceEventHandler interface. -type EventHandler struct { +type EventHandler[object client.Object, request comparable] struct { // ctx stores the context that created the event handler // that is used to propagate cancellation signals to each handler function. ctx context.Context - handler handler.EventHandler - queue workqueue.RateLimitingInterface - predicates []predicate.Predicate -} - -// HandlerFuncs converts EventHandler to a ResourceEventHandlerFuncs -// TODO: switch to ResourceEventHandlerDetailedFuncs with client-go 1.27 -func (e *EventHandler) HandlerFuncs() cache.ResourceEventHandlerFuncs { - return cache.ResourceEventHandlerFuncs{ - AddFunc: e.OnAdd, - UpdateFunc: e.OnUpdate, - DeleteFunc: e.OnDelete, - } + handler handler.TypedEventHandler[object, request] + queue workqueue.TypedRateLimitingInterface[request] + predicates []predicate.TypedPredicate[object] } // OnAdd creates CreateEvent and calls Create on EventHandler. -func (e *EventHandler) OnAdd(obj interface{}) { - c := event.CreateEvent{} +func (e *EventHandler[object, request]) OnAdd(obj interface{}, isInInitialList bool) { + c := event.TypedCreateEvent[object]{ + IsInInitialList: isInInitialList, + } // Pull Object out of the object - if o, ok := obj.(client.Object); ok { + if o, ok := obj.(object); ok { c.Object = o } else { log.Error(nil, "OnAdd missing Object", @@ -89,10 +87,10 @@ func (e *EventHandler) OnAdd(obj interface{}) { } // OnUpdate creates UpdateEvent and calls Update on EventHandler. -func (e *EventHandler) OnUpdate(oldObj, newObj interface{}) { - u := event.UpdateEvent{} +func (e *EventHandler[object, request]) OnUpdate(oldObj, newObj interface{}) { + u := event.TypedUpdateEvent[object]{} - if o, ok := oldObj.(client.Object); ok { + if o, ok := oldObj.(object); ok { u.ObjectOld = o } else { log.Error(nil, "OnUpdate missing ObjectOld", @@ -101,7 +99,7 @@ func (e *EventHandler) OnUpdate(oldObj, newObj interface{}) { } // Pull Object out of the object - if o, ok := newObj.(client.Object); ok { + if o, ok := newObj.(object); ok { u.ObjectNew = o } else { log.Error(nil, "OnUpdate missing ObjectNew", @@ -122,8 +120,8 @@ func (e *EventHandler) OnUpdate(oldObj, newObj interface{}) { } // OnDelete creates DeleteEvent and calls Delete on EventHandler. -func (e *EventHandler) OnDelete(obj interface{}) { - d := event.DeleteEvent{} +func (e *EventHandler[object, request]) OnDelete(obj interface{}) { + d := event.TypedDeleteEvent[object]{} // Deal with tombstone events by pulling the object out. Tombstone events wrap the object in a // DeleteFinalStateUnknown struct, so the object needs to be pulled out. @@ -149,7 +147,7 @@ func (e *EventHandler) OnDelete(obj interface{}) { } // Pull Object out of the object - if o, ok := obj.(client.Object); ok { + if o, ok := obj.(object); ok { d.Object = o } else { log.Error(nil, "OnDelete missing Object", diff --git a/pkg/internal/source/internal_test.go b/pkg/internal/source/internal_test.go index 0574f7180e..73eb1a1d28 100644 --- a/pkg/internal/source/internal_test.go +++ b/pkg/internal/source/internal_test.go @@ -23,9 +23,11 @@ import ( . "github.com/onsi/gomega" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" internal "sigs.k8s.io/controller-runtime/pkg/internal/source" + "sigs.k8s.io/controller-runtime/pkg/reconcile" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,41 +38,40 @@ import ( ) var _ = Describe("Internal", func() { - var ctx = context.Background() - var instance *internal.EventHandler + var instance *internal.EventHandler[client.Object, reconcile.Request] var funcs, setfuncs *handler.Funcs var set bool - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { funcs = &handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect CreateEvent to be called.") }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect DeleteEvent to be called.") }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect UpdateEvent to be called.") }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Did not expect GenericEvent to be called.") }, } setfuncs = &handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { set = true }, } @@ -90,27 +91,27 @@ var _ = Describe("Internal", func() { newPod.Labels = map[string]string{"foo": "bar"} }) - It("should create a CreateEvent", func() { - funcs.CreateFunc = func(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { + It("should create a CreateEvent", func(ctx SpecContext) { + funcs.CreateFunc = func(ctx context.Context, evt event.CreateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(evt.Object).To(Equal(pod)) } - instance.OnAdd(pod) + instance.OnAdd(pod, false) }) - It("should used Predicates to filter CreateEvents", func() { + It("should used Predicates to filter CreateEvents", func(ctx SpecContext) { instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, }) set = false - instance.OnAdd(pod) + instance.OnAdd(pod, false) Expect(set).To(BeFalse()) set = false instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, }) - instance.OnAdd(pod) + instance.OnAdd(pod, false) Expect(set).To(BeTrue()) set = false @@ -118,7 +119,7 @@ var _ = Describe("Internal", func() { predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, }) - instance.OnAdd(pod) + instance.OnAdd(pod, false) Expect(set).To(BeFalse()) set = false @@ -126,7 +127,7 @@ var _ = Describe("Internal", func() { predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, }) - instance.OnAdd(pod) + instance.OnAdd(pod, false) Expect(set).To(BeFalse()) set = false @@ -134,20 +135,20 @@ var _ = Describe("Internal", func() { predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, }) - instance.OnAdd(pod) + instance.OnAdd(pod, false) Expect(set).To(BeTrue()) }) It("should not call Create EventHandler if the object is not a runtime.Object", func() { - instance.OnAdd(&metav1.ObjectMeta{}) + instance.OnAdd(&metav1.ObjectMeta{}, false) }) It("should not call Create EventHandler if the object does not have metadata", func() { - instance.OnAdd(FooRuntimeObject{}) + instance.OnAdd(FooRuntimeObject{}, false) }) - It("should create an UpdateEvent", func() { - funcs.UpdateFunc = func(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + It("should create an UpdateEvent", func(ctx SpecContext) { + funcs.UpdateFunc = func(ctx context.Context, evt event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(evt.ObjectOld).To(Equal(pod)) Expect(evt.ObjectNew).To(Equal(newPod)) @@ -155,7 +156,7 @@ var _ = Describe("Internal", func() { instance.OnUpdate(pod, newPod) }) - It("should used Predicates to filter UpdateEvents", func() { + It("should used Predicates to filter UpdateEvents", func(ctx SpecContext) { set = false instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(updateEvent event.UpdateEvent) bool { return false }}, @@ -206,14 +207,14 @@ var _ = Describe("Internal", func() { }) It("should create a DeleteEvent", func() { - funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(evt.Object).To(Equal(pod)) } instance.OnDelete(pod) }) - It("should used Predicates to filter DeleteEvents", func() { + It("should used Predicates to filter DeleteEvents", func(ctx SpecContext) { set = false instance = internal.NewEventHandler(ctx, &controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return false }}, @@ -262,11 +263,10 @@ var _ = Describe("Internal", func() { }) It("should create a DeleteEvent from a tombstone", func() { - tombstone := cache.DeletedFinalStateUnknown{ Obj: pod, } - funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(evt.Object).To(Equal(pod)) Expect(evt.DeleteStateUnknown).Should(BeTrue()) @@ -280,11 +280,20 @@ var _ = Describe("Internal", func() { instance.OnDelete(tombstone) }) It("should ignore objects without meta", func() { - instance.OnAdd(Foo{}) + instance.OnAdd(Foo{}, false) instance.OnUpdate(Foo{}, Foo{}) instance.OnDelete(Foo{}) }) }) + + Describe("Kind", func() { + It("should return kind source type", func() { + kind := internal.Kind[*corev1.Pod, reconcile.Request]{ + Type: &corev1.Pod{}, + } + Expect(kind.String()).Should(Equal("kind source: *v1.Pod")) + }) + }) }) type Foo struct{} diff --git a/pkg/internal/source/kind.go b/pkg/internal/source/kind.go index b3a8227125..2854244523 100644 --- a/pkg/internal/source/kind.go +++ b/pkg/internal/source/kind.go @@ -4,47 +4,59 @@ import ( "context" "errors" "fmt" + "reflect" "time" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" + toolscache "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" ) +var logKind = logf.RuntimeLog.WithName("source").WithName("Kind") + // Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create). -type Kind struct { +type Kind[object client.Object, request comparable] struct { // Type is the type of object to watch. e.g. &v1.Pod{} - Type client.Object + Type object // Cache used to watch APIs Cache cache.Cache - // started may contain an error if one was encountered during startup. If its closed and does not + Handler handler.TypedEventHandler[object, request] + + Predicates []predicate.TypedPredicate[object] + + // startedErr may contain an error if one was encountered during startup. If its closed and does not // contain an error, startup and syncing finished. - started chan error + startedErr chan error startCancel func() } // Start is internal and should be called only by the Controller to register an EventHandler with the Informer // to enqueue reconcile.Requests. -func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { - if ks.Type == nil { +func (ks *Kind[object, request]) Start(ctx context.Context, queue workqueue.TypedRateLimitingInterface[request]) error { + if isNil(ks.Type) { return fmt.Errorf("must create Kind with a non-nil object") } - if ks.Cache == nil { + if isNil(ks.Cache) { return fmt.Errorf("must create Kind with a non-nil cache") } + if isNil(ks.Handler) { + return errors.New("must create Kind with non-nil handler") + } // cache.GetInformer will block until its context is cancelled if the cache was already started and it can not // sync that informer (most commonly due to RBAC issues). ctx, ks.startCancel = context.WithCancel(ctx) - ks.started = make(chan error) + ks.startedErr = make(chan error, 1) // Buffer chan to not leak goroutines if WaitForSync isn't called go func() { var ( i cache.Informer @@ -60,42 +72,44 @@ func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue w kindMatchErr := &meta.NoKindMatchError{} switch { case errors.As(lastErr, &kindMatchErr): - log.Error(lastErr, "if kind is a CRD, it should be installed before calling Start", + logKind.Error(lastErr, "if kind is a CRD, it should be installed before calling Start", "kind", kindMatchErr.GroupKind) case runtime.IsNotRegisteredError(lastErr): - log.Error(lastErr, "kind must be registered to the Scheme") + logKind.Error(lastErr, "kind must be registered to the Scheme") default: - log.Error(lastErr, "failed to get informer from cache") + logKind.Error(lastErr, "failed to get informer from cache") } return false, nil // Retry. } return true, nil }); err != nil { if lastErr != nil { - ks.started <- fmt.Errorf("failed to get informer from cache: %w", lastErr) + ks.startedErr <- fmt.Errorf("failed to get informer from cache: %w", lastErr) return } - ks.started <- err + ks.startedErr <- err return } - _, err := i.AddEventHandler(NewEventHandler(ctx, queue, handler, prct).HandlerFuncs()) + _, err := i.AddEventHandlerWithOptions(NewEventHandler(ctx, queue, ks.Handler, ks.Predicates), toolscache.HandlerOptions{ + Logger: &logKind, + }) if err != nil { - ks.started <- err + ks.startedErr <- err return } if !ks.Cache.WaitForCacheSync(ctx) { // Would be great to return something more informative here - ks.started <- errors.New("cache did not sync") + ks.startedErr <- errors.New("cache did not sync") } - close(ks.started) + close(ks.startedErr) }() return nil } -func (ks *Kind) String() string { - if ks.Type != nil { +func (ks *Kind[object, request]) String() string { + if !isNil(ks.Type) { return fmt.Sprintf("kind source: %T", ks.Type) } return "kind source: unknown type" @@ -103,9 +117,9 @@ func (ks *Kind) String() string { // WaitForSync implements SyncingSource to allow controllers to wait with starting // workers until the cache is synced. -func (ks *Kind) WaitForSync(ctx context.Context) error { +func (ks *Kind[object, request]) WaitForSync(ctx context.Context) error { select { - case err := <-ks.started: + case err := <-ks.startedErr: return err case <-ctx.Done(): ks.startCancel() @@ -115,3 +129,15 @@ func (ks *Kind) WaitForSync(ctx context.Context) error { return fmt.Errorf("timed out waiting for cache to be synced for Kind %T", ks.Type) } } + +func isNil(arg any) bool { + if v := reflect.ValueOf(arg); !v.IsValid() || ((v.Kind() == reflect.Ptr || + v.Kind() == reflect.Interface || + v.Kind() == reflect.Slice || + v.Kind() == reflect.Map || + v.Kind() == reflect.Chan || + v.Kind() == reflect.Func) && v.IsNil()) { + return true + } + return false +} diff --git a/pkg/internal/syncs/syncs.go b/pkg/internal/syncs/syncs.go new file mode 100644 index 0000000000..c78a30377a --- /dev/null +++ b/pkg/internal/syncs/syncs.go @@ -0,0 +1,38 @@ +package syncs + +import ( + "context" + "reflect" + "sync" +) + +// MergeChans returns a channel that is closed when any of the input channels are signaled. +// The caller must call the returned CancelFunc to ensure no resources are leaked. +func MergeChans[T any](chans ...<-chan T) (<-chan T, context.CancelFunc) { + var once sync.Once + out := make(chan T) + cancel := make(chan T) + cancelFunc := func() { + once.Do(func() { + close(cancel) + }) + <-out + } + cases := make([]reflect.SelectCase, len(chans)+1) + for i := range chans { + cases[i] = reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(chans[i]), + } + } + cases[len(cases)-1] = reflect.SelectCase{ + Dir: reflect.SelectRecv, + Chan: reflect.ValueOf(cancel), + } + go func() { + defer close(out) + _, _, _ = reflect.Select(cases) + }() + + return out, cancelFunc +} diff --git a/pkg/internal/syncs/syncs_test.go b/pkg/internal/syncs/syncs_test.go new file mode 100644 index 0000000000..7bf7d598a0 --- /dev/null +++ b/pkg/internal/syncs/syncs_test.go @@ -0,0 +1,107 @@ +package syncs + +import ( + "testing" + "time" + + // This appears to be needed so that the prow test runner won't fail. + _ "github.com/onsi/ginkgo/v2" + _ "github.com/onsi/gomega" +) + +func TestMergeChans(t *testing.T) { + tests := []struct { + name string + count int + signal int + }{ + { + name: "single channel, close 0", + count: 1, + signal: 0, + }, + { + name: "double channel, close 0", + count: 2, + signal: 0, + }, + { + name: "five channel, close 0", + count: 5, + signal: 0, + }, + { + name: "five channel, close 1", + count: 5, + signal: 1, + }, + { + name: "five channel, close 2", + count: 5, + signal: 2, + }, + { + name: "five channel, close 3", + count: 5, + signal: 3, + }, + { + name: "five channel, close 4", + count: 5, + signal: 4, + }, + { + name: "single channel, cancel", + count: 1, + signal: -1, + }, + { + name: "double channel, cancel", + count: 2, + signal: -1, + }, + { + name: "five channel, cancel", + count: 5, + signal: -1, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if callAndClose(test.count, test.signal, 1) { + t.Error("timeout before merged channel closed") + } + }) + } +} + +func callAndClose(numChans, signalChan, timeoutSeconds int) bool { + chans := make([]chan struct{}, numChans) + readOnlyChans := make([]<-chan struct{}, numChans) + for i := range chans { + chans[i] = make(chan struct{}) + readOnlyChans[i] = chans[i] + } + defer func() { + for i := range chans { + close(chans[i]) + } + }() + + merged, cancel := MergeChans(readOnlyChans...) + defer cancel() + + timer := time.NewTimer(time.Duration(timeoutSeconds) * time.Second) + + if signalChan >= 0 { + chans[signalChan] <- struct{}{} + } else { + cancel() + } + select { + case <-merged: + return false + case <-timer.C: + return true + } +} diff --git a/pkg/internal/testing/OWNERS b/pkg/internal/testing/OWNERS deleted file mode 100644 index 25fda2ebac..0000000000 --- a/pkg/internal/testing/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/devel/owners.md - -approvers: - - testing-integration-approvers diff --git a/pkg/internal/testing/certs/tinyca_test.go b/pkg/internal/testing/certs/tinyca_test.go index 6e0540ba9f..5d84de56fb 100644 --- a/pkg/internal/testing/certs/tinyca_test.go +++ b/pkg/internal/testing/certs/tinyca_test.go @@ -47,8 +47,8 @@ var _ = Describe("TinyCA", func() { }) It("should be usable for signing & verifying", func() { - Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageCertSign).NotTo(Equal(0), "should be usable for cert signing") - Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(Equal(0), "should be usable for signature verifying") + Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageCertSign).NotTo(BeEquivalentTo(0), "should be usable for cert signing") + Expect(ca.CA.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") }) }) @@ -141,8 +141,8 @@ var _ = Describe("TinyCA", func() { cert, err := ca.NewServingCert() Expect(err).NotTo(HaveOccurred(), "should be able to generate a serving cert") - Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(Equal(0), "should be usable for key enciphering") - Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(Equal(0), "should be usable for signature verifying") + Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(BeEquivalentTo(0), "should be usable for key enciphering") + Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") Expect(cert.Cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageServerAuth), "should be usable for server auth") }) @@ -172,8 +172,8 @@ var _ = Describe("TinyCA", func() { }) It("should be usable for client auth, verifying, and enciphering", func() { - Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(Equal(0), "should be usable for key enciphering") - Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(Equal(0), "should be usable for signature verifying") + Expect(cert.Cert.KeyUsage&x509.KeyUsageKeyEncipherment).NotTo(BeEquivalentTo(0), "should be usable for key enciphering") + Expect(cert.Cert.KeyUsage&x509.KeyUsageDigitalSignature).NotTo(BeEquivalentTo(0), "should be usable for signature verifying") Expect(cert.Cert.ExtKeyUsage).To(ContainElement(x509.ExtKeyUsageClientAuth), "should be usable for client auth") }) diff --git a/pkg/internal/testing/controlplane/apiserver.go b/pkg/internal/testing/controlplane/apiserver.go index c9a1a232ea..aadb69e84f 100644 --- a/pkg/internal/testing/controlplane/apiserver.go +++ b/pkg/internal/testing/controlplane/apiserver.go @@ -374,7 +374,12 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - servingCerts, err := ca.NewServingCert() + servingAddresses := []string{"localhost"} + if s.SecureServing.ListenAddr.Address != "" { + servingAddresses = append(servingAddresses, s.SecureServing.ListenAddr.Address) + } + + servingCerts, err := ca.NewServingCert(servingAddresses...) if err != nil { return err } @@ -384,10 +389,10 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0o640); err != nil { return err } - if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0o640); err != nil { return err } @@ -404,10 +409,10 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - if err := os.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0o640); err != nil { return err } - return os.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec + return os.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0o640) } // Stop stops this process gracefully, waits for its termination, and cleans up @@ -421,6 +426,9 @@ func (s *APIServer) Stop() error { return err } } + if s.Authn == nil { + return nil + } return s.Authn.Stop() } diff --git a/pkg/internal/testing/controlplane/apiserver_test.go b/pkg/internal/testing/controlplane/apiserver_test.go index 6ce1577d45..0811e9fb59 100644 --- a/pkg/internal/testing/controlplane/apiserver_test.go +++ b/pkg/internal/testing/controlplane/apiserver_test.go @@ -17,8 +17,13 @@ limitations under the License. package controlplane_test import ( + "crypto/x509" + "encoding/pem" "errors" + "net" "net/url" + "os" + "path" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -128,7 +133,7 @@ var _ = Describe("APIServer", func() { Context("when SecureServing is not set", func() { It("should be defaulted with a random port", func() { - Expect(server.Port).NotTo(Equal(0)) + Expect(server.Port).NotTo(BeEquivalentTo(0)) }) }) }) @@ -191,6 +196,66 @@ var _ = Describe("APIServer", func() { }) }) + // These tests assume that 'localhost' resolves to 127.0.0.1. It can resolve + // to other addresses as well (e.g. ::1 on IPv6), but it must always resolve + // to 127.0.0.1. + Describe(("generated certificates"), func() { + getCertificate := func() *x509.Certificate { + // Read the cert file + certFile := path.Join(server.CertDir, "apiserver.crt") + certBytes, err := os.ReadFile(certFile) + Expect(err).NotTo(HaveOccurred(), "should be able to read the cert file") + + // Decode and parse it + block, remainder := pem.Decode(certBytes) + Expect(block).NotTo(BeNil(), "should be able to decode the cert file") + Expect(remainder).To(BeEmpty(), "should not have any extra data in the cert file") + Expect(block.Type).To(Equal("CERTIFICATE"), "should be a certificate block") + + cert, err := x509.ParseCertificate(block.Bytes) + Expect(err).NotTo(HaveOccurred(), "should be able to parse the cert file") + + return cert + } + + Context("when SecureServing are not set", func() { + It("should have localhost/127.0.0.1 in the certificate altnames", func() { + cert := getCertificate() + + Expect(cert.Subject.CommonName).To(Equal("localhost")) + Expect(cert.DNSNames).To(ConsistOf("localhost")) + expectedIPAddresses := []net.IP{ + net.ParseIP("127.0.0.1").To4(), + net.ParseIP(server.SecureServing.ListenAddr.Address).To4(), + } + Expect(cert.IPAddresses).To(ContainElements(expectedIPAddresses)) + }) + }) + + Context("when SecureServing host & port are set", func() { + BeforeEach(func() { + server.SecureServing = SecureServing{ + ListenAddr: process.ListenAddr{ + Address: "1.2.3.4", + Port: "5678", + }, + } + }) + + It("should have the host in the certificate altnames", func() { + cert := getCertificate() + + Expect(cert.Subject.CommonName).To(Equal("localhost")) + Expect(cert.DNSNames).To(ConsistOf("localhost")) + expectedIPAddresses := []net.IP{ + net.ParseIP("127.0.0.1").To4(), + net.ParseIP(server.SecureServing.ListenAddr.Address).To4(), + } + Expect(cert.IPAddresses).To(ContainElements(expectedIPAddresses)) + }) + }) + }) + Describe("setting up auth", func() { var auth *fakeAuthn BeforeEach(func() { diff --git a/pkg/internal/testing/controlplane/auth.go b/pkg/internal/testing/controlplane/auth.go index 16c86a712c..b44035ebf2 100644 --- a/pkg/internal/testing/controlplane/auth.go +++ b/pkg/internal/testing/controlplane/auth.go @@ -128,7 +128,7 @@ func (c *CertAuthn) Start() error { return fmt.Errorf("start called before configure") } caCrt := c.ca.CA.CertBytes() - if err := os.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { return fmt.Errorf("unable to save the client certificate CA to %s: %w", c.caCrtPath(), err) } diff --git a/pkg/internal/testing/controlplane/etcd.go b/pkg/internal/testing/controlplane/etcd.go index c30d213295..98ffe3ac5e 100644 --- a/pkg/internal/testing/controlplane/etcd.go +++ b/pkg/internal/testing/controlplane/etcd.go @@ -159,6 +159,10 @@ func (e *Etcd) setProcessState() error { // Stop stops this process gracefully, waits for its termination, and cleans up // the DataDir if necessary. func (e *Etcd) Stop() error { + if e.processState == nil { + return nil + } + if e.processState.DirNeedsCleaning { e.DataDir = "" // reset the directory if it was randomly allocated, so that we can safely restart } diff --git a/pkg/internal/testing/controlplane/kubectl.go b/pkg/internal/testing/controlplane/kubectl.go index a27b7a0ff8..a41bb77c4d 100644 --- a/pkg/internal/testing/controlplane/kubectl.go +++ b/pkg/internal/testing/controlplane/kubectl.go @@ -112,6 +112,7 @@ func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) { cmd := exec.Command(k.Path, allArgs...) cmd.Stdout = stdoutBuffer cmd.Stderr = stderrBuffer + cmd.SysProcAttr = process.GetSysProcAttr() err = cmd.Run() diff --git a/pkg/internal/testing/controlplane/plane_test.go b/pkg/internal/testing/controlplane/plane_test.go index cd0359dbca..a228e5a51c 100644 --- a/pkg/internal/testing/controlplane/plane_test.go +++ b/pkg/internal/testing/controlplane/plane_test.go @@ -17,8 +17,6 @@ limitations under the License. package controlplane_test import ( - "context" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" kauthn "k8s.io/api/authorization/v1" @@ -69,7 +67,7 @@ var _ = Describe("Control Plane", func() { Expect(plane.Stop()).To(Succeed()) }) - It("should provision a working legacy user and legacy kubectl", func() { + It("should provision a working legacy user and legacy kubectl", func(ctx SpecContext) { By("grabbing the legacy kubectl") Expect(plane.KubeCtl()).NotTo(BeNil()) @@ -89,7 +87,7 @@ var _ = Describe("Control Plane", func() { }, }, } - Expect(cl.Create(context.Background(), sar)).To(Succeed(), "should be able to make a Self-SAR") + Expect(cl.Create(ctx, sar)).To(Succeed(), "should be able to make a Self-SAR") Expect(sar.Status.Allowed).To(BeTrue(), "admin user should be able to do everything") }) diff --git a/pkg/ratelimiter/doc.go b/pkg/internal/testing/process/procattr_other.go similarity index 55% rename from pkg/ratelimiter/doc.go rename to pkg/internal/testing/process/procattr_other.go index a01d603fe5..df13b341a4 100644 --- a/pkg/ratelimiter/doc.go +++ b/pkg/internal/testing/process/procattr_other.go @@ -1,5 +1,8 @@ +//go:build !aix && !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !zos +// +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!zos + /* -Copyright 2020 The Kubernetes Authors. +Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,9 +17,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Package ratelimiter defines rate limiters used by Controllers to limit how frequently requests may be queued. +package process -Typical rate limiters that can be used are implemented in client-go's workqueue package. -*/ -package ratelimiter +import "syscall" + +// GetSysProcAttr returns the SysProcAttr to use for the process, +// for non-unix systems this returns nil. +func GetSysProcAttr() *syscall.SysProcAttr { + return nil +} diff --git a/pkg/internal/testing/process/procattr_unix.go b/pkg/internal/testing/process/procattr_unix.go new file mode 100644 index 0000000000..83ad509af0 --- /dev/null +++ b/pkg/internal/testing/process/procattr_unix.go @@ -0,0 +1,33 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris || zos +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris zos + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package process + +import ( + "golang.org/x/sys/unix" +) + +// GetSysProcAttr returns the SysProcAttr to use for the process, +// for unix systems this returns a SysProcAttr with Setpgid set to true, +// which inherits the parent's process group id. +func GetSysProcAttr() *unix.SysProcAttr { + return &unix.SysProcAttr{ + Setpgid: true, + } +} diff --git a/pkg/internal/testing/process/process.go b/pkg/internal/testing/process/process.go index af83c70a2f..0d541921e2 100644 --- a/pkg/internal/testing/process/process.go +++ b/pkg/internal/testing/process/process.go @@ -155,6 +155,7 @@ func (ps *State) Start(stdout, stderr io.Writer) (err error) { ps.Cmd = exec.Command(ps.Path, ps.Args...) ps.Cmd.Stdout = stdout ps.Cmd.Stderr = stderr + ps.Cmd.SysProcAttr = GetSysProcAttr() ready := make(chan bool) timedOut := time.After(ps.StartTimeout) @@ -214,7 +215,7 @@ func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh // there's probably certs *somewhere*, // but it's fine to just skip validating // them for health checks during testing - InsecureSkipVerify: true, //nolint:gosec + InsecureSkipVerify: true, }, }, } @@ -265,6 +266,9 @@ func (ps *State) Stop() error { case <-ps.waitDone: break case <-timedOut: + if err := ps.Cmd.Process.Signal(syscall.SIGKILL); err != nil { + return fmt.Errorf("unable to kill process %s: %w", ps.Path, err) + } return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path)) } ps.ready = false diff --git a/pkg/internal/testing/process/process_test.go b/pkg/internal/testing/process/process_test.go index 5b0708227a..1d01e95b09 100644 --- a/pkg/internal/testing/process/process_test.go +++ b/pkg/internal/testing/process/process_test.go @@ -352,16 +352,7 @@ var _ = Describe("Init", func() { }) var simpleBashScript = []string{ - "-c", - ` - i=0 - while true - do - echo "loop $i" >&2 - let 'i += 1' - sleep 0.2 - done - `, + "-c", "tail -f /dev/null", } func getServerURL(server *ghttp.Server) url.URL { diff --git a/pkg/leaderelection/fake/leader_election.go b/pkg/leaderelection/fake/leader_election.go index 5a82cf43b8..ab816a19a7 100644 --- a/pkg/leaderelection/fake/leader_election.go +++ b/pkg/leaderelection/fake/leader_election.go @@ -19,7 +19,9 @@ package fake import ( "context" "encoding/json" + "errors" "os" + "sync/atomic" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,6 +32,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/recorder" ) +// ControllableResourceLockInterface is an interface that extends resourcelock.Interface to be +// controllable. +type ControllableResourceLockInterface interface { + resourcelock.Interface + + // BlockLeaderElection blocks the leader election process when called. It will not be unblocked + // until UnblockLeaderElection is called. + BlockLeaderElection() + + // UnblockLeaderElection unblocks the leader election. + UnblockLeaderElection() +} + // NewResourceLock creates a new ResourceLock for use in testing // leader election. func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { @@ -40,27 +55,31 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op } id = id + "_" + string(uuid.NewUUID()) - return &ResourceLock{ + return &resourceLock{ id: id, record: resourcelock.LeaderElectionRecord{ HolderIdentity: id, - LeaseDurationSeconds: 15, + LeaseDurationSeconds: 1, AcquireTime: metav1.NewTime(time.Now()), - RenewTime: metav1.NewTime(time.Now().Add(15 * time.Second)), + RenewTime: metav1.NewTime(time.Now().Add(1 * time.Second)), LeaderTransitions: 1, }, }, nil } -// ResourceLock implements the ResourceLockInterface. +var _ ControllableResourceLockInterface = &resourceLock{} + +// resourceLock implements the ResourceLockInterface. // By default returns that the current identity holds the lock. -type ResourceLock struct { +type resourceLock struct { id string record resourcelock.LeaderElectionRecord + + blockedLeaderElection atomic.Bool } // Get implements the ResourceLockInterface. -func (f *ResourceLock) Get(ctx context.Context) (*resourcelock.LeaderElectionRecord, []byte, error) { +func (f *resourceLock) Get(ctx context.Context) (*resourcelock.LeaderElectionRecord, []byte, error) { recordBytes, err := json.Marshal(f.record) if err != nil { return nil, nil, err @@ -69,28 +88,49 @@ func (f *ResourceLock) Get(ctx context.Context) (*resourcelock.LeaderElectionRec } // Create implements the ResourceLockInterface. -func (f *ResourceLock) Create(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { +func (f *resourceLock) Create(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { + if f.blockedLeaderElection.Load() { + // If leader election is blocked, we do not allow creating a new record. + return errors.New("leader election is blocked, cannot create new record") + } + f.record = ler return nil } // Update implements the ResourceLockInterface. -func (f *ResourceLock) Update(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { +func (f *resourceLock) Update(ctx context.Context, ler resourcelock.LeaderElectionRecord) error { + if f.blockedLeaderElection.Load() { + // If leader election is blocked, we do not allow updating records + return errors.New("leader election is blocked, cannot update record") + } + f.record = ler + return nil } // RecordEvent implements the ResourceLockInterface. -func (f *ResourceLock) RecordEvent(s string) { +func (f *resourceLock) RecordEvent(s string) { } // Identity implements the ResourceLockInterface. -func (f *ResourceLock) Identity() string { +func (f *resourceLock) Identity() string { return f.id } // Describe implements the ResourceLockInterface. -func (f *ResourceLock) Describe() string { +func (f *resourceLock) Describe() string { return f.id } + +// BlockLeaderElection blocks the leader election process when called. +func (f *resourceLock) BlockLeaderElection() { + f.blockedLeaderElection.Store(true) +} + +// UnblockLeaderElection blocks the leader election process when called. +func (f *resourceLock) UnblockLeaderElection() { + f.blockedLeaderElection.Store(false) +} diff --git a/pkg/leaderelection/leader_election.go b/pkg/leaderelection/leader_election.go index ee4fcf4cbe..6c013e7992 100644 --- a/pkg/leaderelection/leader_election.go +++ b/pkg/leaderelection/leader_election.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "os" + "time" "k8s.io/apimachinery/pkg/util/uuid" coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" @@ -49,6 +50,16 @@ type Options struct { // LeaderElectionID determines the name of the resource that leader election // will use for holding the leader lock. LeaderElectionID string + + // RenewDeadline is the renew deadline for this leader election client. + // Must be set to ensure the resource lock has an appropriate client timeout. + // Without that, a single slow response from the API server can result + // in losing leadership. + RenewDeadline time.Duration + + // LeaderLabels are an optional set of labels that will be set on the lease object + // when this replica becomes leader + LeaderLabels map[string]string } // NewResourceLock creates a new resource lock for use in a leader election loop. @@ -56,7 +67,6 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op if !options.LeaderElection { return nil, nil } - // Default resource lock to "leases". The previous default (from v0.7.0 to v0.11.x) was configmapsleases, which was // used to migrate from configmaps to leases. Since the default was "configmapsleases" for over a year, spanning // five minor releases, any actively maintained operators are very likely to have a released version that uses @@ -86,8 +96,21 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op } id = id + "_" + string(uuid.NewUUID()) + // Construct config for leader election + config = rest.AddUserAgent(config, "leader-election") + + // Timeout set for a client used to contact to Kubernetes should be lower than + // RenewDeadline to keep a single hung request from forcing a leader loss. + // Setting it to max(time.Second, RenewDeadline/2) as a reasonable heuristic. + if options.RenewDeadline != 0 { + timeout := options.RenewDeadline / 2 + if timeout < time.Second { + timeout = time.Second + } + config.Timeout = timeout + } + // Construct clients for leader election - rest.AddUserAgent(config, "leader-election") corev1Client, err := corev1client.NewForConfig(config) if err != nil { return nil, err @@ -98,7 +121,7 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op return nil, err } - return resourcelock.New(options.LeaderElectionResourceLock, + return resourcelock.NewWithLabels(options.LeaderElectionResourceLock, options.LeaderElectionNamespace, options.LeaderElectionID, corev1Client, @@ -106,7 +129,9 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op resourcelock.ResourceLockConfig{ Identity: id, EventRecorder: recorderProvider.GetEventRecorderFor(id), - }) + }, + options.LeaderLabels, + ) } func getInClusterNamespace() (string, error) { diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go index b75604b6be..404a859e4d 100644 --- a/pkg/log/log_test.go +++ b/pkg/log/log_test.go @@ -17,7 +17,6 @@ limitations under the License. package log import ( - "context" "errors" "github.com/go-logr/logr" @@ -194,12 +193,12 @@ var _ = Describe("logging", func() { }() go func() { defer GinkgoRecover() - delegLog.WithValues("with-value") + delegLog.WithValues("key", "with-value") close(withValuesDone) }() go func() { defer GinkgoRecover() - child.WithValues("grandchild") + child.WithValues("key", "grandchild") close(grandChildDone) }() go func() { @@ -297,17 +296,17 @@ var _ = Describe("logging", func() { }) Describe("logger from context", func() { - It("should return default logger when context is empty", func() { - gotLog := FromContext(context.Background()) + It("should return default logger when context is empty", func(ctx SpecContext) { + gotLog := FromContext(ctx) Expect(gotLog).To(Not(BeNil())) }) - It("should return existing logger", func() { + It("should return existing logger", func(specCtx SpecContext) { root := &fakeLoggerRoot{} baseLog := &fakeLogger{root: root} wantLog := logr.New(baseLog).WithName("my-logger") - ctx := IntoContext(context.Background(), wantLog) + ctx := IntoContext(specCtx, wantLog) gotLog := FromContext(ctx) Expect(gotLog).To(Not(BeNil())) @@ -318,12 +317,12 @@ var _ = Describe("logging", func() { )) }) - It("should have added key-values", func() { + It("should have added key-values", func(specCtx SpecContext) { root := &fakeLoggerRoot{} baseLog := &fakeLogger{root: root} wantLog := logr.New(baseLog).WithName("my-logger") - ctx := IntoContext(context.Background(), wantLog) + ctx := IntoContext(specCtx, wantLog) gotLog := FromContext(ctx, "tag1", "value1") Expect(gotLog).To(Not(BeNil())) diff --git a/pkg/log/warning_handler.go b/pkg/log/warning_handler.go index e9522632d3..413b56d2e4 100644 --- a/pkg/log/warning_handler.go +++ b/pkg/log/warning_handler.go @@ -17,13 +17,12 @@ limitations under the License. package log import ( + "context" "sync" - - "github.com/go-logr/logr" ) // KubeAPIWarningLoggerOptions controls the behavior -// of a rest.WarningHandler constructed using NewKubeAPIWarningLogger(). +// of a rest.WarningHandlerWithContext constructed using NewKubeAPIWarningLogger(). type KubeAPIWarningLoggerOptions struct { // Deduplicate indicates a given warning message should only be written once. // Setting this to true in a long-running process handling many warnings can @@ -33,10 +32,8 @@ type KubeAPIWarningLoggerOptions struct { // KubeAPIWarningLogger is a wrapper around // a provided logr.Logger that implements the -// rest.WarningHandler interface. +// rest.WarningHandlerWithContext interface. type KubeAPIWarningLogger struct { - // logger is used to log responses with the warning header - logger logr.Logger // opts contain options controlling warning output opts KubeAPIWarningLoggerOptions // writtenLock gurads written @@ -46,9 +43,11 @@ type KubeAPIWarningLogger struct { written map[string]struct{} } -// HandleWarningHeader handles logging for responses from API server that are -// warnings with code being 299 and uses a logr.Logger for its logging purposes. -func (l *KubeAPIWarningLogger) HandleWarningHeader(code int, agent string, message string) { +// HandleWarningHeaderWithContext handles logging for responses from API server that are +// warnings with code being 299 and uses a logr.Logger from context for its logging purposes. +func (l *KubeAPIWarningLogger) HandleWarningHeaderWithContext(ctx context.Context, code int, _ string, message string) { + log := FromContext(ctx) + if code != 299 || len(message) == 0 { return } @@ -62,13 +61,13 @@ func (l *KubeAPIWarningLogger) HandleWarningHeader(code int, agent string, messa } l.written[message] = struct{}{} } - l.logger.Info(message) + log.Info(message) } -// NewKubeAPIWarningLogger returns an implementation of rest.WarningHandler that logs warnings -// with code = 299 to the provided logr.Logger. -func NewKubeAPIWarningLogger(l logr.Logger, opts KubeAPIWarningLoggerOptions) *KubeAPIWarningLogger { - h := &KubeAPIWarningLogger{logger: l, opts: opts} +// NewKubeAPIWarningLogger returns an implementation of rest.WarningHandlerWithContext that logs warnings +// with code = 299 to the logger passed into HandleWarningHeaderWithContext via the context. +func NewKubeAPIWarningLogger(opts KubeAPIWarningLoggerOptions) *KubeAPIWarningLogger { + h := &KubeAPIWarningLogger{opts: opts} if opts.Deduplicate { h.written = map[string]struct{}{} } diff --git a/pkg/log/zap/flags.go b/pkg/log/zap/flags.go index fb492b14da..4ebac57dcb 100644 --- a/pkg/log/zap/flags.go +++ b/pkg/log/zap/flags.go @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package zap contains helpers for setting up a new logr.Logger instance -// using the Zap logging framework. package zap import ( @@ -32,6 +30,7 @@ var levelStrings = map[string]zapcore.Level{ "debug": zap.DebugLevel, "info": zap.InfoLevel, "error": zap.ErrorLevel, + "panic": zap.PanicLevel, } var stackLevelStrings = map[string]zapcore.Level{ diff --git a/pkg/log/zap/kube_helpers.go b/pkg/log/zap/kube_helpers.go index 3b4ebfdaa0..c47fe6646f 100644 --- a/pkg/log/zap/kube_helpers.go +++ b/pkg/log/zap/kube_helpers.go @@ -78,6 +78,7 @@ func (w kubeObjectWrapper) MarshalLogObject(enc zapcore.ObjectEncoder) error { func (k *KubeAwareEncoder) Clone() zapcore.Encoder { return &KubeAwareEncoder{ Encoder: k.Encoder.Clone(), + Verbose: k.Verbose, } } diff --git a/pkg/log/zap/zap.go b/pkg/log/zap/zap.go index ee89a7c6a4..607b6680d5 100644 --- a/pkg/log/zap/zap.go +++ b/pkg/log/zap/zap.go @@ -148,11 +148,6 @@ type Options struct { // DestWriter controls the destination of the log output. Defaults to // os.Stderr. DestWriter io.Writer - // DestWritter controls the destination of the log output. Defaults to - // os.Stderr. - // - // Deprecated: Use DestWriter instead - DestWritter io.Writer // Level configures the verbosity of the logging. // Defaults to Debug when Development is true and Info otherwise. // A zap log level should be multiplied by -1 to get the logr verbosity. @@ -174,11 +169,8 @@ type Options struct { // addDefaults adds defaults to the Options. func (o *Options) addDefaults() { - if o.DestWriter == nil && o.DestWritter == nil { + if o.DestWriter == nil { o.DestWriter = os.Stderr - } else if o.DestWriter == nil && o.DestWritter != nil { - // while misspelled DestWritter is deprecated but still not removed - o.DestWriter = o.DestWritter } if o.Development { @@ -255,7 +247,7 @@ func NewRaw(opts ...Opts) *zap.Logger { // Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn) // Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) // - zap-encoder: Zap log encoding (one of 'json' or 'console') -// - zap-log-level: Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', +// - zap-log-level: Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic' // or any integer value > 0 which corresponds to custom debug levels of increasing verbosity"). // - zap-stacktrace-level: Zap Level at and above which stacktraces are captured (one of 'info', 'error' or 'panic') // - zap-time-encoding: Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'), @@ -279,7 +271,7 @@ func (o *Options) BindFlags(fs *flag.FlagSet) { o.Level = fromFlag } fs.Var(&levelVal, "zap-log-level", - "Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', "+ + "Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', 'panic'"+ "or any integer value > 0 which corresponds to custom debug levels of increasing verbosity") // Set the StrackTrace Level diff --git a/pkg/log/zap/zap_test.go b/pkg/log/zap/zap_test.go index f7fad41f06..3e80113a65 100644 --- a/pkg/log/zap/zap_test.go +++ b/pkg/log/zap/zap_test.go @@ -327,6 +327,24 @@ var _ = Describe("Zap log level flag options setup", func() { Expect(outRaw).To(BeEmpty()) }) + + It("Should output only panic logs, otherwise empty logs", func() { + args := []string{"--zap-log-level=panic"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) + + logOut := new(bytes.Buffer) + + logger := New(UseFlagOptions(&fromFlags), WriteTo(logOut)) + logger.V(0).Info(logInfoLevel0) + logger.V(1).Info(logDebugLevel1) + logger.V(2).Info(logDebugLevel2) + + outRaw := logOut.Bytes() + + Expect(outRaw).To(BeEmpty()) + }) }) Context("with zap-log-level with increased verbosity.", func() { diff --git a/pkg/manager/example_test.go b/pkg/manager/example_test.go index 06712d7171..02cfa11946 100644 --- a/pkg/manager/example_test.go +++ b/pkg/manager/example_test.go @@ -23,7 +23,6 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client/config" - conf "sigs.k8s.io/controller-runtime/pkg/config" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -61,7 +60,10 @@ func ExampleNew_limitToNamespaces() { mgr, err := manager.New(cfg, manager.Options{ NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) { - opts.Namespaces = []string{"namespace1", "namespace2"} + opts.DefaultNamespaces = map[string]cache.Config{ + "namespace1": {}, + "namespace2": {}, + } return cache.New(config, opts) }}, ) @@ -91,43 +93,3 @@ func ExampleManager_start() { os.Exit(1) } } - -// This example will populate Options from a custom config file -// using defaults. -func ExampleOptions_andFrom() { - opts := manager.Options{} - if _, err := opts.AndFrom(conf.File()); err != nil { - log.Error(err, "unable to load config") - os.Exit(1) - } - - cfg, err := config.GetConfig() - if err != nil { - log.Error(err, "unable to get kubeconfig") - os.Exit(1) - } - - mgr, err := manager.New(cfg, opts) - if err != nil { - log.Error(err, "unable to set up manager") - os.Exit(1) - } - log.Info("created manager", "manager", mgr) -} - -// This example will populate Options from a custom config file -// using defaults and will panic if there are errors. -func ExampleOptions_andFromOrDie() { - cfg, err := config.GetConfig() - if err != nil { - log.Error(err, "unable to get kubeconfig") - os.Exit(1) - } - - mgr, err := manager.New(cfg, manager.Options{}.AndFromOrDie(conf.File())) - if err != nil { - log.Error(err, "unable to set up manager") - os.Exit(1) - } - log.Info("created manager", "manager", mgr) -} diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index f298229e57..a2c3e5324d 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -28,7 +28,6 @@ import ( "time" "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" @@ -44,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" - "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -57,7 +56,6 @@ const ( defaultReadinessEndpoint = "/readyz" defaultLivenessEndpoint = "/healthz" - defaultMetricsEndpoint = "/metrics" ) var _ Runnable = &controllerManager{} @@ -84,11 +82,8 @@ type controllerManager struct { // on shutdown leaderElectionReleaseOnCancel bool - // metricsListener is used to serve prometheus metrics - metricsListener net.Listener - - // metricsExtraHandlers contains extra handlers to register on http server that serves metrics. - metricsExtraHandlers map[string]http.Handler + // metricsServer is used to serve prometheus metrics + metricsServer metricsserver.Server // healthProbeListener is used to serve liveness probe healthProbeListener net.Listener @@ -184,24 +179,20 @@ func (cm *controllerManager) add(r Runnable) error { return cm.runnables.Add(r) } -// AddMetricsExtraHandler adds extra handler served on path to the http server that serves metrics. -func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Handler) error { +// AddMetricsServerExtraHandler adds extra handler served on path to the http server that serves metrics. +func (cm *controllerManager) AddMetricsServerExtraHandler(path string, handler http.Handler) error { cm.Lock() defer cm.Unlock() - if cm.started { return fmt.Errorf("unable to add new metrics handler because metrics endpoint has already been created") } - - if path == defaultMetricsEndpoint { - return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint) + if cm.metricsServer == nil { + cm.GetLogger().Info("warn: metrics server is currently disabled, registering extra handler will be ignored", "path", path) + return nil } - - if _, found := cm.metricsExtraHandlers[path]; found { - return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path) + if err := cm.metricsServer.AddExtraHandler(path, handler); err != nil { + return err } - - cm.metricsExtraHandlers[path] = handler cm.logger.V(2).Info("Registering metrics http server extra handler", "path", path) return nil } @@ -296,31 +287,10 @@ func (cm *controllerManager) GetControllerOptions() config.Controller { return cm.controllerConfig } -func (cm *controllerManager) addMetricsServer() error { +func (cm *controllerManager) addHealthProbeServer() error { mux := http.NewServeMux() srv := httpserver.New(mux) - handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ - ErrorHandling: promhttp.HTTPErrorOnError, - }) - // TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics - mux.Handle(defaultMetricsEndpoint, handler) - for path, extraHandler := range cm.metricsExtraHandlers { - mux.Handle(path, extraHandler) - } - - return cm.add(&server{ - Kind: "metrics", - Log: cm.logger.WithValues("path", defaultMetricsEndpoint), - Server: srv, - Listener: cm.metricsListener, - }) -} - -func (cm *controllerManager) serveHealthProbes() { - mux := http.NewServeMux() - server := httpserver.New(mux) - if cm.readyzHandler != nil { mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) // Append '/' suffix to handle subpaths @@ -332,7 +302,11 @@ func (cm *controllerManager) serveHealthProbes() { mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) } - go cm.httpServe("health probe", cm.logger, server, cm.healthProbeListener) + return cm.add(&Server{ + Name: "health probe", + Server: srv, + Listener: cm.healthProbeListener, + }) } func (cm *controllerManager) addPprofServer() error { @@ -345,50 +319,13 @@ func (cm *controllerManager) addPprofServer() error { mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) mux.HandleFunc("/debug/pprof/trace", pprof.Trace) - return cm.add(&server{ - Kind: "pprof", - Log: cm.logger, + return cm.add(&Server{ + Name: "pprof", Server: srv, Listener: cm.pprofListener, }) } -func (cm *controllerManager) httpServe(kind string, log logr.Logger, server *http.Server, ln net.Listener) { - log = log.WithValues("kind", kind, "addr", ln.Addr()) - - go func() { - log.Info("Starting server") - if err := server.Serve(ln); err != nil { - if errors.Is(err, http.ErrServerClosed) { - return - } - if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 { - // There might be cases where connections are still open and we try to shutdown - // but not having enough time to close the connection causes an error in Serve - // - // In that case we want to avoid returning an error to the main error channel. - log.Error(err, "error on Serve after stop has been engaged") - return - } - cm.errChan <- err - } - }() - - // Shutdown the server when stop is closed. - <-cm.internalProceduresStop - if err := server.Shutdown(cm.shutdownCtx); err != nil { - if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - // Avoid logging context related errors. - return - } - if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 { - cm.logger.Error(err, "error on Shutdown after stop has been engaged") - return - } - cm.errChan <- err - } -} - // Start starts the manager and waits indefinitely. // There is only two ways to have start return: // An error has occurred during in one of the internal operations, @@ -414,6 +351,16 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { // Initialize the internal context. cm.internalCtx, cm.internalCancel = context.WithCancel(ctx) + // Leader elector must be created before defer that contains engageStopProcedure function + // https://github.com/kubernetes-sigs/controller-runtime/issues/2873 + var leaderElector *leaderelection.LeaderElector + if cm.resourceLock != nil { + leaderElector, err = cm.initLeaderElector() + if err != nil { + return fmt.Errorf("failed during initialization leader election process: %w", err) + } + } + // This chan indicates that stop is complete, in other words all runnables have returned or timeout on stop request stopComplete := make(chan struct{}) defer close(stopComplete) @@ -441,15 +388,19 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { // Metrics should be served whether the controller is leader or not. // (If we don't serve metrics for non-leaders, prometheus will still scrape // the pod but will get a connection refused). - if cm.metricsListener != nil { - if err := cm.addMetricsServer(); err != nil { + if cm.metricsServer != nil { + // Note: We are adding the metrics server directly to HTTPServers here as matching on the + // metricsserver.Server interface in cm.runnables.Add would be very brittle. + if err := cm.runnables.HTTPServers.Add(cm.metricsServer, nil); err != nil { return fmt.Errorf("failed to add metrics server: %w", err) } } // Serve health probes. if cm.healthProbeListener != nil { - cm.serveHealthProbes() + if err := cm.addHealthProbeServer(); err != nil { + return fmt.Errorf("failed to add health probe server: %w", err) + } } // Add pprof server @@ -459,49 +410,63 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { } } - // First start any webhook servers, which includes conversion, validation, and defaulting + // First start any HTTP servers, which includes health probes, metrics and profiling if enabled. + // + // WARNING: HTTPServers includes the health probes, which MUST start before any cache is populated, otherwise + // it would block conversion webhooks to be ready for serving which make the cache never get ready. + logCtx := logr.NewContext(cm.internalCtx, cm.logger) + if err := cm.runnables.HTTPServers.Start(logCtx); err != nil { + return fmt.Errorf("failed to start HTTP servers: %w", err) + } + + // Start any webhook servers, which includes conversion, validation, and defaulting // webhooks that are registered. // // WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition // between conversion webhooks and the cache sync (usually initial list) which causes the webhooks // to never start because no cache can be populated. if err := cm.runnables.Webhooks.Start(cm.internalCtx); err != nil { - if err != nil { - return fmt.Errorf("failed to start webhooks: %w", err) - } + return fmt.Errorf("failed to start webhooks: %w", err) } // Start and wait for caches. if err := cm.runnables.Caches.Start(cm.internalCtx); err != nil { - if err != nil { - return fmt.Errorf("failed to start caches: %w", err) - } + return fmt.Errorf("failed to start caches: %w", err) } // Start the non-leaderelection Runnables after the cache has synced. if err := cm.runnables.Others.Start(cm.internalCtx); err != nil { - if err != nil { - return fmt.Errorf("failed to start other runnables: %w", err) - } + return fmt.Errorf("failed to start other runnables: %w", err) + } + + // Start WarmupRunnables and wait for warmup to complete. + if err := cm.runnables.Warmup.Start(cm.internalCtx); err != nil { + return fmt.Errorf("failed to start warmup runnables: %w", err) } // Start the leader election and all required runnables. { - ctx, cancel := context.WithCancel(context.Background()) + // Create a context that inherits all keys from the parent context + // but can be cancelled independently for leader election management + baseCtx := context.WithoutCancel(ctx) + leaderCtx, cancel := context.WithCancel(baseCtx) cm.leaderElectionCancel = cancel - go func() { - if cm.resourceLock != nil { - if err := cm.startLeaderElection(ctx); err != nil { - cm.errChan <- err - } - } else { + if leaderElector != nil { + // Start the leader elector process + go func() { + leaderElector.Run(leaderCtx) + <-leaderCtx.Done() + close(cm.leaderElectionStopped) + }() + } else { + go func() { // Treat not having leader election enabled the same as being elected. if err := cm.startLeaderElectionRunnables(); err != nil { cm.errChan <- err } close(cm.elected) - } - }() + }() + } } ready = true @@ -550,8 +515,8 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e cm.internalCancel() }) select { - case err, ok := <-cm.errChan: - if ok { + case err := <-cm.errChan: + if !errors.Is(err, context.Canceled) { cm.logger.Error(err, "error received after stop sequence was engaged") } case <-stopComplete: @@ -577,12 +542,26 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e }() go func() { + go func() { + // Stop the warmup runnables in a separate goroutine to avoid blocking. + // It is important to stop the warmup runnables in parallel with the other runnables + // since we cannot assume ordering of whether or not one of the warmup runnables or one + // of the other runnables is holding a lock. + // Cancelling the wrong runnable (one that is not holding the lock) will cause the + // shutdown sequence to block indefinitely as it will wait for the runnable that is + // holding the lock to finish. + cm.logger.Info("Stopping and waiting for warmup runnables") + cm.runnables.Warmup.StopAndWait(cm.shutdownCtx) + }() + // First stop the non-leader election runnables. cm.logger.Info("Stopping and waiting for non leader election runnables") cm.runnables.Others.StopAndWait(cm.shutdownCtx) // Stop all the leader election runnables, which includes reconcilers. cm.logger.Info("Stopping and waiting for leader election runnables") + // Prevent leader election when shutting down a non-elected manager + cm.runnables.LeaderElection.startOnce.Do(func() {}) cm.runnables.LeaderElection.StopAndWait(cm.shutdownCtx) // Stop the caches before the leader election runnables, this is an important @@ -591,10 +570,13 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e cm.logger.Info("Stopping and waiting for caches") cm.runnables.Caches.StopAndWait(cm.shutdownCtx) - // Webhooks should come last, as they might be still serving some requests. + // Webhooks and internal HTTP servers should come last, as they might be still serving some requests. cm.logger.Info("Stopping and waiting for webhooks") cm.runnables.Webhooks.StopAndWait(cm.shutdownCtx) + cm.logger.Info("Stopping and waiting for HTTP servers") + cm.runnables.HTTPServers.StopAndWait(cm.shutdownCtx) + // Proceed to close the manager and overall shutdown context. cm.logger.Info("Wait completed, proceeding to shutdown the manager") shutdownCancel() @@ -615,12 +597,8 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e return nil } -func (cm *controllerManager) startLeaderElectionRunnables() error { - return cm.runnables.LeaderElection.Start(cm.internalCtx) -} - -func (cm *controllerManager) startLeaderElection(ctx context.Context) (err error) { - l, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ +func (cm *controllerManager) initLeaderElector() (*leaderelection.LeaderElector, error) { + leaderElector, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ Lock: cm.resourceLock, LeaseDuration: cm.leaseDuration, RenewDeadline: cm.renewDeadline, @@ -650,16 +628,14 @@ func (cm *controllerManager) startLeaderElection(ctx context.Context) (err error Name: cm.leaderElectionID, }) if err != nil { - return err + return nil, err } - // Start the leader elector process - go func() { - l.Run(ctx) - <-ctx.Done() - close(cm.leaderElectionStopped) - }() - return nil + return leaderElector, nil +} + +func (cm *controllerManager) startLeaderElectionRunnables() error { + return cm.runnables.LeaderElection.Start(cm.internalCtx) } func (cm *controllerManager) Elected() <-chan struct{} { diff --git a/pkg/manager/internal/integration/api/v1/driver_types.go b/pkg/manager/internal/integration/api/v1/driver_types.go new file mode 100644 index 0000000000..9182ed4cc8 --- /dev/null +++ b/pkg/manager/internal/integration/api/v1/driver_types.go @@ -0,0 +1,93 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// Driver is a test type. +type Driver struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` +} + +// DriverList is a list of Drivers. +type DriverList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Driver `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Driver{}, &DriverList{}) +} + +// DeepCopyInto deep copies into the given Driver. +func (d *Driver) DeepCopyInto(out *Driver) { + *out = *d + out.TypeMeta = d.TypeMeta + d.ObjectMeta.DeepCopyInto(&out.ObjectMeta) +} + +// DeepCopy returns a copy of Driver. +func (d *Driver) DeepCopy() *Driver { + if d == nil { + return nil + } + out := new(Driver) + d.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of Driver as runtime.Object. +func (d *Driver) DeepCopyObject() runtime.Object { + return d.DeepCopy() +} + +// DeepCopyInto deep copies into the given DriverList. +func (in *DriverList) DeepCopyInto(out *DriverList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Driver, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy returns a copy of DriverList. +func (in *DriverList) DeepCopy() *DriverList { + if in == nil { + return nil + } + out := new(DriverList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of DriverList as runtime.Object. +func (in *DriverList) DeepCopyObject() runtime.Object { + return in.DeepCopy() +} + +// Hub marks Driver as a Hub for conversion. +func (*Driver) Hub() {} diff --git a/examples/configfile/custom/v1alpha1/types.go b/pkg/manager/internal/integration/api/v1/groupversion_info.go similarity index 51% rename from examples/configfile/custom/v1alpha1/types.go rename to pkg/manager/internal/integration/api/v1/groupversion_info.go index 79e8422c5c..3986a6d023 100644 --- a/examples/configfile/custom/v1alpha1/types.go +++ b/pkg/manager/internal/integration/api/v1/groupversion_info.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,41 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package v1alpha1 provides the CustomControllerManagerConfiguration used for -// demoing componentconfig -// +kubebuilder:object:generate=true -package v1alpha1 +package v1 import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" - cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "examples.x-k8s.io", Version: "v1alpha1"} + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "crew.example.com", Version: "v1"} - // SchemeBuilder is used to add go types to the GroupVersionKind scheme + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) - -// +kubebuilder:object:root=true - -// CustomControllerManagerConfiguration is the Schema for the CustomControllerManagerConfigurations API -type CustomControllerManagerConfiguration struct { - metav1.TypeMeta `json:",inline"` - - // ControllerManagerConfigurationSpec returns the contfigurations for controllers - cfg.ControllerManagerConfigurationSpec `json:",inline"` - - ClusterName string `json:"clusterName,omitempty"` -} - -func init() { - SchemeBuilder.Register(&CustomControllerManagerConfiguration{}) -} diff --git a/pkg/manager/internal/integration/api/v2/driver_types.go b/pkg/manager/internal/integration/api/v2/driver_types.go new file mode 100644 index 0000000000..64012ac749 --- /dev/null +++ b/pkg/manager/internal/integration/api/v2/driver_types.go @@ -0,0 +1,111 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/conversion" + v1 "sigs.k8s.io/controller-runtime/pkg/manager/internal/integration/api/v1" +) + +// Driver is a test type. +type Driver struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` +} + +// DriverList is a list of Drivers. +type DriverList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Driver `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Driver{}, &DriverList{}) +} + +// DeepCopyInto deep copies into the given Driver. +func (d *Driver) DeepCopyInto(out *Driver) { + *out = *d + out.TypeMeta = d.TypeMeta + d.ObjectMeta.DeepCopyInto(&out.ObjectMeta) +} + +// DeepCopy returns a copy of Driver. +func (d *Driver) DeepCopy() *Driver { + if d == nil { + return nil + } + out := new(Driver) + d.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of Driver as runtime.Object. +func (d *Driver) DeepCopyObject() runtime.Object { + return d.DeepCopy() +} + +// DeepCopyInto deep copies into the given DriverList. +func (in *DriverList) DeepCopyInto(out *DriverList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Driver, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy returns a copy of DriverList. +func (in *DriverList) DeepCopy() *DriverList { + if in == nil { + return nil + } + out := new(DriverList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject returns a copy of DriverList as runtime.Object. +func (in *DriverList) DeepCopyObject() runtime.Object { + return in.DeepCopy() +} + +// ConvertTo converts Driver to the Hub version of driver. +func (d *Driver) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*v1.Driver) + dst.Name = d.Name + dst.Namespace = d.Namespace + dst.UID = d.UID + return nil +} + +// ConvertFrom converts Driver from the Hub version of driver. +func (d *Driver) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*v1.Driver) + d.Name = src.Name + d.Namespace = src.Namespace + d.UID = src.UID + return nil +} diff --git a/pkg/manager/internal/integration/api/v2/groupversion_info.go b/pkg/manager/internal/integration/api/v2/groupversion_info.go new file mode 100644 index 0000000000..218a742793 --- /dev/null +++ b/pkg/manager/internal/integration/api/v2/groupversion_info.go @@ -0,0 +1,34 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "crew.example.com", Version: "v2"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/hack/tools/tools.go b/pkg/manager/internal/integration/manager_suite_test.go similarity index 67% rename from hack/tools/tools.go rename to pkg/manager/internal/integration/manager_suite_test.go index 481a7c6f01..1a5a20d5a5 100644 --- a/hack/tools/tools.go +++ b/pkg/manager/internal/integration/manager_suite_test.go @@ -1,7 +1,5 @@ -// +build tools - /* -Copyright 2019 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,10 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -// This package imports things required by build scripts, to force `go mod` to see them as dependencies -package tools +package integration import ( - _ "github.com/joelanford/go-apidiff" - _ "sigs.k8s.io/controller-tools/cmd/controller-gen" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) + +func TestManager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Manager Integration Suite") +} diff --git a/pkg/manager/internal/integration/manager_test.go b/pkg/manager/internal/integration/manager_test.go new file mode 100644 index 0000000000..c83eead3c1 --- /dev/null +++ b/pkg/manager/internal/integration/manager_test.go @@ -0,0 +1,306 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "net" + "net/http" + "reflect" + "sync/atomic" + "time" + "unsafe" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" + crewv1 "sigs.k8s.io/controller-runtime/pkg/manager/internal/integration/api/v1" + crewv2 "sigs.k8s.io/controller-runtime/pkg/manager/internal/integration/api/v2" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/conversion" +) + +var ( + scheme = runtime.NewScheme() + + driverCRD = &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "drivers.crew.example.com", + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: crewv1.GroupVersion.Group, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Plural: "drivers", + Singular: "driver", + Kind: "Driver", + }, + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: crewv1.GroupVersion.Version, + Served: true, + // v1 will be the storage version. + // Reconciler and index will use v2 so we can validate the conversion webhook works. + Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + }, + }, + }, + { + Name: crewv2.GroupVersion.Version, + Served: true, + Storage: false, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + }, + }, + }, + }, + }, + } + + ctx = ctrl.SetupSignalHandler() +) + +var _ = Describe("manger.Manager Start", func() { + // This test ensure the Manager starts without running into any deadlocks as it can be very tricky + // to start health probes, webhooks, caches (including informers) and reconcilers in the right order. + // + // To verify this we set up a test environment in the following state: + // * Ensure Informer sync requires a functioning conversion webhook (and thus readiness probe) + // * Driver CRD is deployed with v1 as storage version + // * A Driver CR is created and stored in the v1 version + // * Setup manager: + // * Set up health probes + // * Set up a Driver v2 reconciler to verify reconciliation works + // * Set up a conversion webhook which only works if readiness probe succeeds (just like via a Kubernetes service) + // * Add an index on v2 Driver to ensure we start and wait for an informer during cache.Start (as part of manager.Start) + // * Note: cache.Start would fail if the conversion webhook doesn't work (which in turn depends on the readiness probe) + // * Note: Adding the index for v2 ensures the Driver list call during Informer sync goes through conversion. + DescribeTable("should start all components without deadlock", func(warmupEnabled bool) { + // Set up schema. + Expect(clientgoscheme.AddToScheme(scheme)).To(Succeed()) + Expect(apiextensionsv1.AddToScheme(scheme)).To(Succeed()) + Expect(crewv1.AddToScheme(scheme)).To(Succeed()) + Expect(crewv2.AddToScheme(scheme)).To(Succeed()) + + // Set up test environment. + env := &envtest.Environment{ + Scheme: scheme, + CRDInstallOptions: envtest.CRDInstallOptions{ + CRDs: []*apiextensionsv1.CustomResourceDefinition{driverCRD}, + }, + } + // Note: The test env configures a conversion webhook on driverCRD during Start. + cfg, err := env.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + defer func() { + Expect(env.Stop()).To(Succeed()) + }() + c, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + + // Create driver CR (which is stored as v1). + driverV1 := &unstructured.Unstructured{} + driverV1.SetGroupVersionKind(crewv1.GroupVersion.WithKind("Driver")) + driverV1.SetName("driver1") + driverV1.SetNamespace(metav1.NamespaceDefault) + Expect(c.Create(ctx, driverV1)).To(Succeed()) + + // Set up Manager. + ctrl.SetLogger(zap.New()) + mgr, err := manager.New(env.Config, manager.Options{ + Scheme: scheme, + HealthProbeBindAddress: ":0", + // Disable metrics to avoid port conflicts. + Metrics: metricsserver.Options{BindAddress: "0"}, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: env.WebhookInstallOptions.LocalServingPort, + Host: env.WebhookInstallOptions.LocalServingHost, + CertDir: env.WebhookInstallOptions.LocalServingCertDir, + }), + }) + Expect(err).NotTo(HaveOccurred()) + + // Configure health probes. + Expect(mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed()) + Expect(mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker())).To(Succeed()) + + // Set up Driver reconciler (using v2). + driverReconciler := &DriverReconciler{ + Client: mgr.GetClient(), + } + Expect( + ctrl.NewControllerManagedBy(mgr). + For(&crewv2.Driver{}). + Named(fmt.Sprintf("driver_warmup_%t", warmupEnabled)). + WithOptions(controller.Options{EnableWarmup: ptr.To(warmupEnabled)}). + Complete(driverReconciler), + ).To(Succeed()) + + // Set up a conversion webhook. + conversionWebhook := createConversionWebhook(mgr) + mgr.GetWebhookServer().Register("/convert", conversionWebhook) + + // Add an index on Driver (using v2). + // Note: This triggers the creation of an Informer for Driver v2. + Expect(mgr.GetCache().IndexField(ctx, &crewv2.Driver{}, "name", func(object client.Object) []string { + return []string{object.GetName()} + })).To(Succeed()) + + // Start the Manager. + ctx, cancel := context.WithCancel(ctx) + go func() { + defer GinkgoRecover() + Expect(mgr.Start(ctx)).To(Succeed()) + }() + + // Verify manager.Start successfully started health probes, webhooks, caches (including informers) and reconcilers. + // Notes: + // * The cache will only start successfully if the informer for v2 Driver is synced. + // * The informer for v2 Driver will only sync if a list on v2 Driver succeeds (which requires a working conversion webhook) + select { + case <-time.After(30 * time.Second): + // Don't wait forever if the manager doesn't come up. + Fail("Manager didn't start in time") + case <-mgr.Elected(): + } + + // Verify the reconciler reconciles. + Eventually(func(g Gomega) { + g.Expect(atomic.LoadUint64(&driverReconciler.ReconcileCount)).Should(BeNumerically(">", 0)) + }, 10*time.Second).Should(Succeed()) + + // Verify conversion webhook was called. + Expect(atomic.LoadUint64(&conversionWebhook.ConversionCount)).Should(BeNumerically(">", 0)) + + // Verify the conversion webhook works by getting the Driver as v1 and v2. + Expect(c.Get(ctx, client.ObjectKeyFromObject(driverV1), driverV1)).To(Succeed()) + driverV2 := &unstructured.Unstructured{} + driverV2.SetGroupVersionKind(crewv2.GroupVersion.WithKind("Driver")) + driverV2.SetName("driver1") + driverV2.SetNamespace(metav1.NamespaceDefault) + Expect(c.Get(ctx, client.ObjectKeyFromObject(driverV2), driverV2)).To(Succeed()) + + // Shutdown the server + cancel() + }, + Entry("controller warmup enabled", true), + Entry("controller warmup not enabled", false), + ) +}) + +type DriverReconciler struct { + Client client.Client + ReconcileCount uint64 +} + +func (r *DriverReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + log := ctrl.LoggerFrom(ctx) + log.Info("Reconciling") + + // Fetch the Driver instance. + cluster := &crewv2.Driver{} + if err := r.Client.Get(ctx, req.NamespacedName, cluster); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + atomic.AddUint64(&r.ReconcileCount, 1) + + return reconcile.Result{}, nil +} + +// ConversionWebhook is just a shim around the conversion handler from +// the webhook package. We use it to simulate the behavior of a conversion +// webhook in a real cluster, i.e. the conversion webhook only works after the +// controller Pod is ready (the readiness probe is up). +type ConversionWebhook struct { + httpClient http.Client + conversionHandler http.Handler + readinessEndpoint string + ConversionCount uint64 +} + +func createConversionWebhook(mgr manager.Manager) *ConversionWebhook { + conversionHandler := conversion.NewWebhookHandler(mgr.GetScheme()) + httpClient := http.Client{ + // Setting a timeout to not get stuck when calling the readiness probe. + Timeout: 5 * time.Second, + } + + // Read the unexported healthProbeListener field of the manager to get the listener address. + // This is a hack but it's better than using a hard-coded port. + v := reflect.ValueOf(mgr).Elem() + field := v.FieldByName("healthProbeListener") + healthProbeListener := *(*net.Listener)(unsafe.Pointer(field.UnsafeAddr())) + readinessEndpoint := fmt.Sprint("http://", healthProbeListener.Addr().String(), "/readyz") + + return &ConversionWebhook{ + httpClient: httpClient, + conversionHandler: conversionHandler, + readinessEndpoint: readinessEndpoint, + } +} + +func (c *ConversionWebhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { + resp, err := c.httpClient.Get(c.readinessEndpoint) + if err != nil { + logf.Log.WithName("conversion-webhook").Error(err, "failed to serve conversion: readiness endpoint is not up") + w.WriteHeader(http.StatusInternalServerError) + return + } + + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + // This simulates the behavior in Kubernetes that conversion webhooks are only served after + // the controller is ready (and thus the Kubernetes service sends requests to the controller). + logf.Log.WithName("conversion-webhook").Info("failed to serve conversion: controller is not ready yet") + w.WriteHeader(http.StatusInternalServerError) + return + } + + atomic.AddUint64(&c.ConversionCount, 1) + c.conversionHandler.ServeHTTP(w, r) +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 7e65ef0c3a..e0e94245e7 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -18,32 +18,31 @@ package manager import ( "context" - "crypto/tls" + "errors" "fmt" "net" "net/http" - "reflect" "time" "github.com/go-logr/logr" + coordinationv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/recorder" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -65,12 +64,14 @@ type Manager interface { // election was configured. Elected() <-chan struct{} - // AddMetricsExtraHandler adds an extra handler served on path to the http server that serves metrics. - // Might be useful to register some diagnostic endpoints e.g. pprof. Note that these endpoints meant to be - // sensitive and shouldn't be exposed publicly. - // If the simple path -> handler mapping offered here is not enough, a new http server/listener should be added as - // Runnable to the manager via Add method. - AddMetricsExtraHandler(path string, handler http.Handler) error + // AddMetricsServerExtraHandler adds an extra handler served on path to the http server that serves metrics. + // Might be useful to register some diagnostic endpoints e.g. pprof. + // + // Note that these endpoints are meant to be sensitive and shouldn't be exposed publicly. + // + // If the simple path -> handler mapping offered here is not enough, + // a new http server/listener should be added as Runnable to the manager via Add method. + AddMetricsServerExtraHandler(path string, handler http.Handler) error // AddHealthzCheck allows you to add Healthz checker AddHealthzCheck(name string, check healthz.Checker) error @@ -140,35 +141,6 @@ type Options struct { // Only use a custom NewClient if you know what you are doing. NewClient client.NewClientFunc - // SyncPeriod determines the minimum frequency at which watched resources are - // reconciled. A lower period will correct entropy more quickly, but reduce - // responsiveness to change if there are many watched resources. Change this - // value only if you know what you are doing. Defaults to 10 hours if unset. - // there will a 10 percent jitter between the SyncPeriod of all controllers - // so that all controllers will not send list requests simultaneously. - // - // This applies to all controllers. - // - // A period sync happens for two reasons: - // 1. To insure against a bug in the controller that causes an object to not - // be requeued, when it otherwise should be requeued. - // 2. To insure against an unknown bug in controller-runtime, or its dependencies, - // that causes an object to not be requeued, when it otherwise should be - // requeued, or to be removed from the queue, when it otherwise should not - // be removed. - // - // If you want - // 1. to insure against missed watch events, or - // 2. to poll services that cannot be watched, - // then we recommend that, instead of changing the default period, the - // controller requeue, with a constant duration `t`, whenever the controller - // is "done" with an object, and would otherwise not requeue it, i.e., we - // recommend the `Reconcile` function return `reconcile.Result{RequeueAfter: t}`, - // instead of `reconcile.Result{}`. - // - // Deprecated: Use Cache.SyncPeriod instead. - SyncPeriod *time.Duration - // Logger is the logger that should be used by this manager. // If none is set, it defaults to log.Log global logger. Logger logr.Logger @@ -229,37 +201,32 @@ type Options struct { // LeaseDuration time first. LeaderElectionReleaseOnCancel bool + // LeaderElectionLabels allows a controller to supplement all leader election api calls with a set of custom labels based on + // the replica attempting to acquire leader status. + LeaderElectionLabels map[string]string + // LeaderElectionResourceLockInterface allows to provide a custom resourcelock.Interface that was created outside // of the controller-runtime. If this value is set the options LeaderElectionID, LeaderElectionNamespace, - // LeaderElectionResourceLock, LeaseDuration, RenewDeadline and RetryPeriod will be ignored. This can be useful if you - // want to use a locking mechanism that is currently not supported, like a MultiLock across two Kubernetes clusters. + // LeaderElectionResourceLock, LeaseDuration, RenewDeadline, RetryPeriod and LeaderElectionLeases will be ignored. + // This can be useful if you want to use a locking mechanism that is currently not supported, like a MultiLock across + // two Kubernetes clusters. LeaderElectionResourceLockInterface resourcelock.Interface // LeaseDuration is the duration that non-leader candidates will // wait to force acquire leadership. This is measured against time of // last observed ack. Default is 15 seconds. LeaseDuration *time.Duration + // RenewDeadline is the duration that the acting controlplane will retry // refreshing leadership before giving up. Default is 10 seconds. RenewDeadline *time.Duration + // RetryPeriod is the duration the LeaderElector clients should wait // between tries of actions. Default is 2 seconds. RetryPeriod *time.Duration - // Namespace, if specified, restricts the manager's cache to watch objects in - // the desired namespace. Defaults to all namespaces. - // - // Note: If a namespace is specified, controllers can still Watch for a - // cluster-scoped resource (e.g Node). For namespaced resources, the cache - // will only hold objects from the desired namespace. - // - // Deprecated: Use Cache.Namespaces instead. - Namespace string - - // MetricsBindAddress is the TCP address that the controller should bind to - // for serving prometheus metrics. - // It can be set to "0" to disable the metrics serving. - MetricsBindAddress string + // Metrics are the metricsserver.Options that will be used to create the metricsserver.Server. + Metrics metricsserver.Options // HealthProbeBindAddress is the TCP address that the controller should bind to // for serving health probes @@ -279,34 +246,9 @@ type Options struct { // before exposing it to public. PprofBindAddress string - // Port is the port that the webhook server serves at. - // It is used to set webhook.Server.Port if WebhookServer is not set. - // - // Deprecated: Use WebhookServer instead. A WebhookServer can be created via webhook.NewServer. - Port int - // Host is the hostname that the webhook server binds to. - // It is used to set webhook.Server.Host if WebhookServer is not set. - // - // Deprecated: Use WebhookServer instead. A WebhookServer can be created via webhook.NewServer. - Host string - - // CertDir is the directory that contains the server key and certificate. - // If not set, webhook server would look up the server key and certificate in - // {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate - // must be named tls.key and tls.crt, respectively. - // It is used to set webhook.Server.CertDir if WebhookServer is not set. - // - // Deprecated: Use WebhookServer instead. A WebhookServer can be created via webhook.NewServer. - CertDir string - - // TLSOpts is used to allow configuring the TLS config used for the webhook server. - // - // Deprecated: Use WebhookServer instead. A WebhookServer can be created via webhook.NewServer. - TLSOpts []func(*tls.Config) - // WebhookServer is an externally configured webhook.Server. By default, - // a Manager will create a default server using Port, Host, and CertDir; - // if this is set, the Manager will use this server instead. + // a Manager will create a server via webhook.NewServer with default settings. + // If this is set, the Manager will use this server instead. WebhookServer webhook.Server // BaseContext is the function that provides Context values to Runnables @@ -314,18 +256,6 @@ type Options struct { // will receive a new Background Context instead. BaseContext BaseContextFunc - // ClientDisableCacheFor tells the client that, if any cache is used, to bypass it - // for the given objects. - // - // Deprecated: Use Client.Cache.DisableCacheFor instead. - ClientDisableCacheFor []client.Object - - // DryRunClient specifies whether the client should be configured to enforce - // dryRun mode. - // - // Deprecated: Use Client.DryRun instead. - DryRunClient bool - // EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API // Use this to customize the event correlator and spam filter // @@ -353,7 +283,7 @@ type Options struct { // Dependency injection for testing newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) - newMetricsListener func(addr string) (net.Listener, error) + newMetricsServer func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) newHealthProbeListener func(addr string) (net.Listener, error) newPprofListener func(addr string) (net.Listener, error) } @@ -389,8 +319,23 @@ type LeaderElectionRunnable interface { NeedLeaderElection() bool } +// warmupRunnable knows if a Runnable requires warmup. A warmup runnable is a runnable +// that should be run when the manager is started but before it becomes leader. +// Note: Implementing this interface is only useful when LeaderElection can be enabled, as the +// behavior when leaderelection is not enabled is to run LeaderElectionRunnables immediately. +type warmupRunnable interface { + // Warmup will be called when the manager is started but before it becomes leader. + Warmup(context.Context) error +} + // New returns a new Manager for creating Controllers. +// Note that if ContentType in the given config is not set, "application/vnd.kubernetes.protobuf" +// will be used for all built-in resources of Kubernetes, and "application/json" is for other types +// including all CRD resources. func New(config *rest.Config, options Options) (Manager, error) { + if config == nil { + return nil, errors.New("must specify Config") + } // Set default values for options fields options = setOptionsDefaults(options) @@ -398,20 +343,21 @@ func New(config *rest.Config, options Options) (Manager, error) { clusterOptions.Scheme = options.Scheme clusterOptions.MapperProvider = options.MapperProvider clusterOptions.Logger = options.Logger - clusterOptions.SyncPeriod = options.SyncPeriod clusterOptions.NewCache = options.NewCache clusterOptions.NewClient = options.NewClient clusterOptions.Cache = options.Cache clusterOptions.Client = options.Client - clusterOptions.Namespace = options.Namespace //nolint:staticcheck - clusterOptions.ClientDisableCacheFor = options.ClientDisableCacheFor //nolint:staticcheck - clusterOptions.DryRunClient = options.DryRunClient //nolint:staticcheck - clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck + clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck }) if err != nil { return nil, err } + config = rest.CopyConfig(config) + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + // Create the recorder provider to inject event recorders for the components. // TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific // to the particular controller that it's being injected into, rather than a generic one like is here. @@ -429,7 +375,20 @@ func New(config *rest.Config, options Options) (Manager, error) { leaderRecorderProvider = recorderProvider } else { leaderConfig = rest.CopyConfig(options.LeaderElectionConfig) - leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, cluster.GetHTTPClient(), cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster) + scheme := cluster.GetScheme() + err := corev1.AddToScheme(scheme) + if err != nil { + return nil, err + } + err = coordinationv1.AddToScheme(scheme) + if err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(options.LeaderElectionConfig) + if err != nil { + return nil, err + } + leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, httpClient, scheme, options.Logger.WithName("events"), options.makeBroadcaster) if err != nil { return nil, err } @@ -444,22 +403,20 @@ func New(config *rest.Config, options Options) (Manager, error) { LeaderElectionResourceLock: options.LeaderElectionResourceLock, LeaderElectionID: options.LeaderElectionID, LeaderElectionNamespace: options.LeaderElectionNamespace, + RenewDeadline: *options.RenewDeadline, + LeaderLabels: options.LeaderElectionLabels, }) if err != nil { return nil, err } } - // Create the metrics listener. This will throw an error if the metrics bind - // address is invalid or already in use. - metricsListener, err := options.newMetricsListener(options.MetricsBindAddress) + // Create the metrics server. + metricsServer, err := options.newMetricsServer(options.Metrics, config, cluster.GetHTTPClient()) if err != nil { return nil, err } - // By default we have no extra endpoints to expose on metrics http server. - metricsExtraHandlers := make(map[string]http.Handler) - // Create health probes listener. This will throw an error if the bind // address is invalid or already in use. healthProbeListener, err := options.newHealthProbeListener(options.HealthProbeBindAddress) @@ -474,18 +431,16 @@ func New(config *rest.Config, options Options) (Manager, error) { return nil, fmt.Errorf("failed to new pprof listener: %w", err) } - errChan := make(chan error) - runnables := newRunnables(options.BaseContext, errChan) - + errChan := make(chan error, 1) + runnables := newRunnables(options.BaseContext, errChan).withLogger(options.Logger) return &controllerManager{ - stopProcedureEngaged: pointer.Int64(0), + stopProcedureEngaged: ptr.To(int64(0)), cluster: cluster, runnables: runnables, errChan: errChan, recorderProvider: recorderProvider, resourceLock: resourceLock, - metricsListener: metricsListener, - metricsExtraHandlers: metricsExtraHandlers, + metricsServer: metricsServer, controllerConfig: options.Controller, logger: options.Logger, elected: make(chan struct{}), @@ -505,131 +460,6 @@ func New(config *rest.Config, options Options) (Manager, error) { }, nil } -// AndFrom will use a supplied type and convert to Options -// any options already set on Options will be ignored, this is used to allow -// cli flags to override anything specified in the config file. -// -// Deprecated: This function has been deprecated and will be removed in a future release, -// The Component Configuration package has been unmaintained for over a year and is no longer -// actively developed. Users should migrate to their own configuration format -// and configure Manager.Options directly. -// See https://github.com/kubernetes-sigs/controller-runtime/issues/895 -// for more information, feedback, and comments. -func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, error) { - newObj, err := loader.Complete() - if err != nil { - return o, err - } - - o = o.setLeaderElectionConfig(newObj) - - if o.SyncPeriod == nil && newObj.SyncPeriod != nil { - o.SyncPeriod = &newObj.SyncPeriod.Duration - } - - if o.Namespace == "" && newObj.CacheNamespace != "" { - o.Namespace = newObj.CacheNamespace - } - - if o.MetricsBindAddress == "" && newObj.Metrics.BindAddress != "" { - o.MetricsBindAddress = newObj.Metrics.BindAddress - } - - if o.HealthProbeBindAddress == "" && newObj.Health.HealthProbeBindAddress != "" { - o.HealthProbeBindAddress = newObj.Health.HealthProbeBindAddress - } - - if o.ReadinessEndpointName == "" && newObj.Health.ReadinessEndpointName != "" { - o.ReadinessEndpointName = newObj.Health.ReadinessEndpointName - } - - if o.LivenessEndpointName == "" && newObj.Health.LivenessEndpointName != "" { - o.LivenessEndpointName = newObj.Health.LivenessEndpointName - } - - if o.Port == 0 && newObj.Webhook.Port != nil { - o.Port = *newObj.Webhook.Port - } - if o.Host == "" && newObj.Webhook.Host != "" { - o.Host = newObj.Webhook.Host - } - if o.CertDir == "" && newObj.Webhook.CertDir != "" { - o.CertDir = newObj.Webhook.CertDir - } - if o.WebhookServer == nil { - o.WebhookServer = webhook.NewServer(webhook.Options{ - Port: o.Port, - Host: o.Host, - CertDir: o.CertDir, - }) - } - - if newObj.Controller != nil { - if o.Controller.CacheSyncTimeout == 0 && newObj.Controller.CacheSyncTimeout != nil { - o.Controller.CacheSyncTimeout = *newObj.Controller.CacheSyncTimeout - } - - if len(o.Controller.GroupKindConcurrency) == 0 && len(newObj.Controller.GroupKindConcurrency) > 0 { - o.Controller.GroupKindConcurrency = newObj.Controller.GroupKindConcurrency - } - } - - return o, nil -} - -// AndFromOrDie will use options.AndFrom() and will panic if there are errors. -// -// Deprecated: This function has been deprecated and will be removed in a future release, -// The Component Configuration package has been unmaintained for over a year and is no longer -// actively developed. Users should migrate to their own configuration format -// and configure Manager.Options directly. -// See https://github.com/kubernetes-sigs/controller-runtime/issues/895 -// for more information, feedback, and comments. -func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Options { - o, err := o.AndFrom(loader) - if err != nil { - panic(fmt.Sprintf("could not parse config file: %v", err)) - } - return o -} - -func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options { - if obj.LeaderElection == nil { - // The source does not have any configuration; noop - return o - } - - if !o.LeaderElection && obj.LeaderElection.LeaderElect != nil { - o.LeaderElection = *obj.LeaderElection.LeaderElect - } - - if o.LeaderElectionResourceLock == "" && obj.LeaderElection.ResourceLock != "" { - o.LeaderElectionResourceLock = obj.LeaderElection.ResourceLock - } - - if o.LeaderElectionNamespace == "" && obj.LeaderElection.ResourceNamespace != "" { - o.LeaderElectionNamespace = obj.LeaderElection.ResourceNamespace - } - - if o.LeaderElectionID == "" && obj.LeaderElection.ResourceName != "" { - o.LeaderElectionID = obj.LeaderElection.ResourceName - } - - if o.LeaseDuration == nil && !reflect.DeepEqual(obj.LeaderElection.LeaseDuration, metav1.Duration{}) { - o.LeaseDuration = &obj.LeaderElection.LeaseDuration.Duration - } - - if o.RenewDeadline == nil && !reflect.DeepEqual(obj.LeaderElection.RenewDeadline, metav1.Duration{}) { - o.RenewDeadline = &obj.LeaderElection.RenewDeadline.Duration - } - - if o.RetryPeriod == nil && !reflect.DeepEqual(obj.LeaderElection.RetryPeriod, metav1.Duration{}) { - o.RetryPeriod = &obj.LeaderElection.RetryPeriod.Duration - } - - return o -} - // defaultHealthProbeListener creates the default health probes listener bound to the given address. func defaultHealthProbeListener(addr string) (net.Listener, error) { if addr == "" || addr == "0" { @@ -688,8 +518,8 @@ func setOptionsDefaults(options Options) Options { } } - if options.newMetricsListener == nil { - options.newMetricsListener = metrics.NewListener + if options.newMetricsServer == nil { + options.newMetricsServer = metricsserver.NewServer } leaseDuration, renewDeadline, retryPeriod := defaultLeaseDuration, defaultRenewDeadline, defaultRetryPeriod if options.LeaseDuration == nil { @@ -729,17 +559,16 @@ func setOptionsDefaults(options Options) Options { options.Logger = log.Log } + if options.Controller.Logger.GetSink() == nil { + options.Controller.Logger = options.Logger + } + if options.BaseContext == nil { options.BaseContext = defaultBaseContext } if options.WebhookServer == nil { - options.WebhookServer = webhook.NewServer(webhook.Options{ - Host: options.Host, - Port: options.Port, - CertDir: options.CertDir, - TLSOpts: options.TLSOpts, - }) + options.WebhookServer = webhook.NewServer(webhook.Options{}) } return options diff --git a/pkg/manager/manager_options_test.go b/pkg/manager/manager_options_test.go deleted file mode 100644 index 3718bedcbe..0000000000 --- a/pkg/manager/manager_options_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package manager - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/config" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - configv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" -) - -var _ = Describe("manager.Options", func() { - Describe("AndFrom", func() { - Describe("reading custom type using OfKind", func() { - var ( - o Options - c customConfig - err error - ) - - JustBeforeEach(func() { - s := runtime.NewScheme() - o = Options{Scheme: s} - c = customConfig{} - - _, err = o.AndFrom(config.File().AtPath("./testdata/custom-config.yaml").OfKind(&c)) - }) - - It("should not panic or fail", func() { - Expect(err).To(Succeed()) - }) - It("should set custom properties", func() { - Expect(c.CustomValue).To(Equal("foo")) - }) - }) - }) -}) - -type customConfig struct { - metav1.TypeMeta `json:",inline"` - configv1alpha1.ControllerManagerConfigurationSpec `json:",inline"` - CustomValue string `json:"customValue"` -} - -func (in *customConfig) DeepCopyObject() runtime.Object { - out := &customConfig{} - *out = *in - - in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) - - return out -} diff --git a/pkg/manager/manager_suite_test.go b/pkg/manager/manager_suite_test.go index ab514ef1e9..7fbf7184ac 100644 --- a/pkg/manager/manager_suite_test.go +++ b/pkg/manager/manager_suite_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) func TestSource(t *testing.T) { @@ -69,12 +69,12 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) // Prevent the metrics listener being created - metrics.DefaultBindAddress = "0" + metricsserver.DefaultBindAddress = "0" }) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) // Put the DefaultBindAddress back - metrics.DefaultBindAddress = ":8080" + metricsserver.DefaultBindAddress = ":8080" }) diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 0186f46f8b..4363d62f59 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -18,7 +18,6 @@ package manager import ( "context" - "crypto/tls" "errors" "fmt" "io" @@ -30,28 +29,27 @@ import ( "time" "github.com/go-logr/logr" + "github.com/go-logr/logr/funcr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" "go.uber.org/goleak" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" - - configv1alpha1 "k8s.io/component-base/config/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake" "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/recorder" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -119,141 +117,6 @@ var _ = Describe("manger.Manager", func() { Expect(err.Error()).To(ContainSubstring("expected error")) }) - It("should be able to load Options from cfg.ControllerManagerConfiguration type", func() { - duration := metav1.Duration{Duration: 48 * time.Hour} - port := int(6090) - leaderElect := false - - ccfg := &v1alpha1.ControllerManagerConfiguration{ - ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ - SyncPeriod: &duration, - LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ - LeaderElect: &leaderElect, - ResourceLock: "leases", - ResourceNamespace: "default", - ResourceName: "ctrl-lease", - LeaseDuration: duration, - RenewDeadline: duration, - RetryPeriod: duration, - }, - CacheNamespace: "default", - Metrics: v1alpha1.ControllerMetrics{ - BindAddress: ":6000", - }, - Health: v1alpha1.ControllerHealth{ - HealthProbeBindAddress: "6060", - ReadinessEndpointName: "/readyz", - LivenessEndpointName: "/livez", - }, - Webhook: v1alpha1.ControllerWebhook{ - Port: &port, - Host: "localhost", - CertDir: "/certs", - }, - }, - } - - m, err := Options{}.AndFrom(&fakeDeferredLoader{ccfg}) - Expect(err).ToNot(HaveOccurred()) - - Expect(*m.SyncPeriod).To(Equal(duration.Duration)) - Expect(m.LeaderElection).To(Equal(leaderElect)) - Expect(m.LeaderElectionResourceLock).To(Equal("leases")) - Expect(m.LeaderElectionNamespace).To(Equal("default")) - Expect(m.LeaderElectionID).To(Equal("ctrl-lease")) - Expect(m.LeaseDuration.String()).To(Equal(duration.Duration.String())) - Expect(m.RenewDeadline.String()).To(Equal(duration.Duration.String())) - Expect(m.RetryPeriod.String()).To(Equal(duration.Duration.String())) - Expect(m.Namespace).To(Equal("default")) - Expect(m.MetricsBindAddress).To(Equal(":6000")) - Expect(m.HealthProbeBindAddress).To(Equal("6060")) - Expect(m.ReadinessEndpointName).To(Equal("/readyz")) - Expect(m.LivenessEndpointName).To(Equal("/livez")) - Expect(m.Port).To(Equal(port)) - Expect(m.Host).To(Equal("localhost")) - Expect(m.CertDir).To(Equal("/certs")) - }) - - It("should be able to keep Options when cfg.ControllerManagerConfiguration set", func() { - optDuration := time.Duration(2) - duration := metav1.Duration{Duration: 48 * time.Hour} - port := int(6090) - leaderElect := false - - ccfg := &v1alpha1.ControllerManagerConfiguration{ - ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ - SyncPeriod: &duration, - LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ - LeaderElect: &leaderElect, - ResourceLock: "leases", - ResourceNamespace: "default", - ResourceName: "ctrl-lease", - LeaseDuration: duration, - RenewDeadline: duration, - RetryPeriod: duration, - }, - CacheNamespace: "default", - Metrics: v1alpha1.ControllerMetrics{ - BindAddress: ":6000", - }, - Health: v1alpha1.ControllerHealth{ - HealthProbeBindAddress: "6060", - ReadinessEndpointName: "/readyz", - LivenessEndpointName: "/livez", - }, - Webhook: v1alpha1.ControllerWebhook{ - Port: &port, - Host: "localhost", - CertDir: "/certs", - }, - }, - } - - optionsTlSOptsFuncs := []func(*tls.Config){ - func(config *tls.Config) {}, - } - m, err := Options{ - SyncPeriod: &optDuration, - LeaderElection: true, - LeaderElectionResourceLock: "configmaps", - LeaderElectionNamespace: "ctrl", - LeaderElectionID: "ctrl-configmap", - LeaseDuration: &optDuration, - RenewDeadline: &optDuration, - RetryPeriod: &optDuration, - Namespace: "ctrl", - MetricsBindAddress: ":7000", - HealthProbeBindAddress: "5000", - ReadinessEndpointName: "/readiness", - LivenessEndpointName: "/liveness", - WebhookServer: webhook.NewServer(webhook.Options{ - Port: 8080, - Host: "example.com", - CertDir: "/pki", - TLSOpts: optionsTlSOptsFuncs, - }), - }.AndFrom(&fakeDeferredLoader{ccfg}) - Expect(err).ToNot(HaveOccurred()) - - Expect(m.SyncPeriod.String()).To(Equal(optDuration.String())) - Expect(m.LeaderElection).To(BeTrue()) - Expect(m.LeaderElectionResourceLock).To(Equal("configmaps")) - Expect(m.LeaderElectionNamespace).To(Equal("ctrl")) - Expect(m.LeaderElectionID).To(Equal("ctrl-configmap")) - Expect(m.LeaseDuration.String()).To(Equal(optDuration.String())) - Expect(m.RenewDeadline.String()).To(Equal(optDuration.String())) - Expect(m.RetryPeriod.String()).To(Equal(optDuration.String())) - Expect(m.Namespace).To(Equal("ctrl")) - Expect(m.MetricsBindAddress).To(Equal(":7000")) - Expect(m.HealthProbeBindAddress).To(Equal("5000")) - Expect(m.ReadinessEndpointName).To(Equal("/readiness")) - Expect(m.LivenessEndpointName).To(Equal("/liveness")) - Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Port).To(Equal(8080)) - Expect(m.WebhookServer.(*webhook.DefaultServer).Options.Host).To(Equal("example.com")) - Expect(m.WebhookServer.(*webhook.DefaultServer).Options.CertDir).To(Equal("/pki")) - Expect(m.WebhookServer.(*webhook.DefaultServer).Options.TLSOpts).To(Equal(optionsTlSOptsFuncs)) - }) - It("should lazily initialize a webhook server if needed", func() { By("creating a manager with options") m, err := New(cfg, Options{WebhookServer: webhook.NewServer(webhook.Options{Port: 9440, Host: "foo.com"})}) @@ -267,23 +130,10 @@ var _ = Describe("manger.Manager", func() { Expect(svr.(*webhook.DefaultServer).Options.Host).To(Equal("foo.com")) }) - It("should lazily initialize a webhook server if needed (deprecated)", func() { - By("creating a manager with options") - m, err := New(cfg, Options{Port: 9440, Host: "foo.com"}) - Expect(err).NotTo(HaveOccurred()) - Expect(m).NotTo(BeNil()) - - By("checking options are passed to the webhook server") - svr := m.GetWebhookServer() - Expect(svr).NotTo(BeNil()) - Expect(svr.(*webhook.DefaultServer).Options.Port).To(Equal(9440)) - Expect(svr.(*webhook.DefaultServer).Options.Host).To(Equal("foo.com")) - }) - It("should not initialize a webhook server if Options.WebhookServer is set", func() { By("creating a manager with options") srv := webhook.NewServer(webhook.Options{Port: 9440}) - m, err := New(cfg, Options{Port: 9441, WebhookServer: srv}) + m, err := New(cfg, Options{WebhookServer: srv}) Expect(err).NotTo(HaveOccurred()) Expect(m).NotTo(BeNil()) @@ -310,17 +160,20 @@ var _ = Describe("manger.Manager", func() { }) Context("with leader election enabled", func() { - It("should only cancel the leader election after all runnables are done", func() { + It("should only cancel the leader election after all runnables are done", func(specCtx SpecContext) { m, err := New(cfg, Options{ LeaderElection: true, LeaderElectionNamespace: "default", LeaderElectionID: "test-leader-election-id-2", HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, PprofBindAddress: "0", }) Expect(err).ToNot(HaveOccurred()) - + gvkcorev1 := schema.GroupVersionKind{Group: corev1.SchemeGroupVersion.Group, Version: corev1.SchemeGroupVersion.Version, Kind: "ConfigMap"} + gvkcoordinationv1 := schema.GroupVersionKind{Group: coordinationv1.SchemeGroupVersion.Group, Version: coordinationv1.SchemeGroupVersion.Version, Kind: "Lease"} + Expect(m.GetScheme().Recognizes(gvkcorev1)).To(BeTrue()) + Expect(m.GetScheme().Recognizes(gvkcoordinationv1)).To(BeTrue()) runnableDone := make(chan struct{}) slowRunnable := RunnableFunc(func(ctx context.Context) error { <-ctx.Done() @@ -337,7 +190,7 @@ var _ = Describe("manger.Manager", func() { close(leaderElectionDone) } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) mgrDone := make(chan struct{}) go func() { defer GinkgoRecover() @@ -356,19 +209,17 @@ var _ = Describe("manger.Manager", func() { <-mgrDone }) - It("should disable gracefulShutdown when stopping to lead", func() { + It("should disable gracefulShutdown when stopping to lead", func(ctx SpecContext) { m, err := New(cfg, Options{ LeaderElection: true, LeaderElectionNamespace: "default", LeaderElectionID: "test-leader-election-id-3", HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, PprofBindAddress: "0", }) Expect(err).ToNot(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() mgrDone := make(chan struct{}) go func() { defer GinkgoRecover() @@ -385,7 +236,8 @@ var _ = Describe("manger.Manager", func() { Expect(cm.gracefulShutdownTimeout.Nanoseconds()).To(Equal(int64(0))) }) - It("should default ID to controller-runtime if ID is not set", func() { + + It("should prevent leader election when shutting down a non-elected manager", func(specCtx SpecContext) { var rl resourcelock.Interface m1, err := New(cfg, Options{ LeaderElection: true, @@ -397,7 +249,7 @@ var _ = Describe("manger.Manager", func() { return rl, err }, HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, PprofBindAddress: "0", }) Expect(err).ToNot(HaveOccurred()) @@ -418,7 +270,105 @@ var _ = Describe("manger.Manager", func() { return rl, err }, HealthProbeBindAddress: "0", - MetricsBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m2).ToNot(BeNil()) + Expect(rl.Describe()).To(Equal("default/test-leader-election-id")) + + m1done := make(chan struct{}) + Expect(m1.Add(RunnableFunc(func(ctx context.Context) error { + defer GinkgoRecover() + close(m1done) + return nil + }))).To(Succeed()) + + go func() { + defer GinkgoRecover() + Expect(m1.Elected()).ShouldNot(BeClosed()) + Expect(m1.Start(specCtx)).NotTo(HaveOccurred()) + }() + <-m1.Elected() + <-m1done + + electionRunnable := &needElection{make(chan struct{})} + + Expect(m2.Add(electionRunnable)).To(Succeed()) + + ctx2, cancel2 := context.WithCancel(specCtx) + m2done := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(m2.Start(ctx2)).NotTo(HaveOccurred()) + close(m2done) + }() + Consistently(m2.Elected()).ShouldNot(Receive()) + + go func() { + defer GinkgoRecover() + Consistently(electionRunnable.ch).ShouldNot(Receive()) + }() + cancel2() + <-m2done + }) + + It("should default RenewDeadline for leader election config", func() { + var rl resourcelock.Interface + m1, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-id", + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + if options.RenewDeadline != 10*time.Second { + return nil, fmt.Errorf("expected RenewDeadline to be 10s, got %v", options.RenewDeadline) + } + var err error + rl, err = leaderelection.NewResourceLock(config, recorderProvider, options) + return rl, err + }, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m1).ToNot(BeNil()) + }) + + It("should default ID to controller-runtime if ID is not set", func(specCtx SpecContext) { + var rl resourcelock.Interface + m1, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-id", + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + var err error + rl, err = leaderelection.NewResourceLock(config, recorderProvider, options) + return rl, err + }, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m1).ToNot(BeNil()) + Expect(rl.Describe()).To(Equal("default/test-leader-election-id")) + + m1cm, ok := m1.(*controllerManager) + Expect(ok).To(BeTrue()) + m1cm.onStoppedLeading = func() {} + + m2, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-id", + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + var err error + rl, err = leaderelection.NewResourceLock(config, recorderProvider, options) + return rl, err + }, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, PprofBindAddress: "0", }) Expect(err).ToNot(HaveOccurred()) @@ -436,12 +386,10 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx1, cancel1 := context.WithCancel(context.Background()) - defer cancel1() go func() { defer GinkgoRecover() Expect(m1.Elected()).ShouldNot(BeClosed()) - Expect(m1.Start(ctx1)).NotTo(HaveOccurred()) + Expect(m1.Start(specCtx)).NotTo(HaveOccurred()) }() <-m1.Elected() <-c1 @@ -453,7 +401,7 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx2, cancel := context.WithCancel(context.Background()) + ctx2, cancel := context.WithCancel(specCtx) m2done := make(chan struct{}) go func() { defer GinkgoRecover() @@ -511,7 +459,7 @@ var _ = Describe("manger.Manager", func() { _, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock) Expect(isLeaseLock).To(BeTrue()) }) - It("should release lease if ElectionReleaseOnCancel is true", func() { + It("should release lease if ElectionReleaseOnCancel is true", func(specCtx SpecContext) { var rl resourcelock.Interface m, err := New(cfg, Options{ LeaderElection: true, @@ -527,7 +475,7 @@ var _ = Describe("manger.Manager", func() { }) Expect(err).ToNot(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) doneCh := make(chan struct{}) go func() { defer GinkgoRecover() @@ -538,12 +486,29 @@ var _ = Describe("manger.Manager", func() { cancel() <-doneCh - ctx, cancel = context.WithCancel(context.Background()) - defer cancel() - record, _, err := rl.Get(ctx) + record, _, err := rl.Get(specCtx) Expect(err).ToNot(HaveOccurred()) Expect(record.HolderIdentity).To(BeEmpty()) }) + It("should set the leaselocks's label field when LeaderElectionLabels is set", func() { + labels := map[string]string{"my-key": "my-val"} + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaderElectionID: "controller-runtime", + LeaderElectionNamespace: "default", + LeaderElectionLabels: labels, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m).ToNot(BeNil()) + cm, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + ll, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock) + Expect(isLeaseLock).To(BeTrue()) + val, exists := ll.Labels["my-key"] + Expect(exists).To(BeTrue()) + Expect(val).To(Equal("my-val")) + }) When("using a custom LeaderElectionResourceLockInterface", func() { It("should use the custom LeaderElectionResourceLockInterface", func() { rl, err := fakeleaderelection.NewResourceLock(nil, nil, leaderelection.Options{}) @@ -565,40 +530,107 @@ var _ = Describe("manger.Manager", func() { }) }) - It("should create a listener for the metrics if a valid address is provided", func() { - var listener net.Listener + It("should create a metrics server if a valid address is provided", func(specCtx SpecContext) { + var srv metricsserver.Server m, err := New(cfg, Options{ - MetricsBindAddress: ":0", - newMetricsListener: func(addr string) (net.Listener, error) { + Metrics: metricsserver.Options{BindAddress: ":0"}, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { var err error - listener, err = metrics.NewListener(addr) - return listener, err + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err }, }) Expect(m).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) - Expect(listener).ToNot(BeNil()) - Expect(listener.Close()).ToNot(HaveOccurred()) + Expect(srv).ToNot(BeNil()) + + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + ctx, cancel := context.WithTimeout(specCtx, 5*time.Second) + Expect(srv.Start(ctx)).To(Succeed()) + cancel() }) - It("should return an error if the metrics bind address is already in use", func() { - ln, err := metrics.NewListener(":0") + It("should create a metrics server if a valid address is provided and secure serving is enabled", func(specCtx SpecContext) { + var srv metricsserver.Server + m, err := New(cfg, Options{ + Metrics: metricsserver.Options{BindAddress: ":0", SecureServing: true}, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { + var err error + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(srv).ToNot(BeNil()) + + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + ctx, cancel := context.WithTimeout(specCtx, 5*time.Second) + Expect(srv.Start(ctx)).To(Succeed()) + cancel() + }) + + It("should be able to create a manager with a cache that fails on missing informer", func() { + m, err := New(cfg, Options{ + Cache: cache.Options{ + ReaderFailOnMissingInformer: true, + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should return an error if the metrics bind address is already in use", func(ctx SpecContext) { + ln, err := net.Listen("tcp", ":0") Expect(err).ShouldNot(HaveOccurred()) - var listener net.Listener + var srv metricsserver.Server m, err := New(cfg, Options{ - MetricsBindAddress: ln.Addr().String(), - newMetricsListener: func(addr string) (net.Listener, error) { + Metrics: metricsserver.Options{ + BindAddress: ln.Addr().String(), + }, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { var err error - listener, err = metrics.NewListener(addr) - return listener, err + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err }, }) - Expect(m).To(BeNil()) - Expect(err).To(HaveOccurred()) - Expect(listener).To(BeNil()) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) - Expect(ln.Close()).ToNot(HaveOccurred()) + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + Expect(srv.Start(ctx)).ToNot(Succeed()) + + Expect(ln.Close()).To(Succeed()) + }) + + It("should return an error if the metrics bind address is already in use and secure serving enabled", func(ctx SpecContext) { + ln, err := net.Listen("tcp", ":0") + Expect(err).ShouldNot(HaveOccurred()) + + var srv metricsserver.Server + m, err := New(cfg, Options{ + Metrics: metricsserver.Options{ + BindAddress: ln.Addr().String(), + SecureServing: true, + }, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { + var err error + srv, err = metricsserver.NewServer(options, config, httpClient) + return srv, err + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + + // Triggering the metric server start here manually to test if it works. + // Usually this happens later during manager.Start(). + Expect(srv.Start(ctx)).ToNot(Succeed()) + + Expect(ln.Close()).To(Succeed()) }) It("should create a listener for the health probes if a valid address is provided", func() { @@ -640,7 +672,7 @@ var _ = Describe("manger.Manager", func() { Describe("Start", func() { var startSuite = func(options Options, callbacks ...func(Manager)) { - It("should Start each Component", func() { + It("should Start each Component", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -660,8 +692,6 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Elected()).ShouldNot(BeClosed()) @@ -691,18 +721,18 @@ var _ = Describe("manger.Manager", func() { Expect(m.GetConfig()).To(Equal(originalCfg)) }) - It("should stop when context is cancelled", func() { + It("should stop when context is cancelled", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { cb(m) } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }) - It("should return an error if it can't start the cache", func() { + It("should return an error if it can't start the cache", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -714,12 +744,10 @@ var _ = Describe("manger.Manager", func() { &cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}}, )).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() Expect(m.Start(ctx)).To(MatchError(ContainSubstring("expected error"))) }) - It("should start the cache before starting anything else", func() { + It("should start the cache before starting anything else", func(ctx SpecContext) { fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { return fakeCache, nil @@ -741,8 +769,6 @@ var _ = Describe("manger.Manager", func() { }) Expect(m.Add(runnable)).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).ToNot(HaveOccurred()) @@ -751,7 +777,7 @@ var _ = Describe("manger.Manager", func() { <-runnableWasStarted }) - It("should start additional clusters before anything else", func() { + It("should start additional clusters before anything else", func(ctx SpecContext) { fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { return fakeCache, nil @@ -784,8 +810,6 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).ToNot(HaveOccurred()) @@ -794,7 +818,7 @@ var _ = Describe("manger.Manager", func() { <-runnableWasStarted }) - It("should return an error if any Components fail to Start", func() { + It("should return an error if any Components fail to Start", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -817,15 +841,12 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - defer GinkgoRecover() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() err = m.Start(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("expected error")) }) - It("should start caches added after Manager has started", func() { + It("should start caches added after Manager has started", func(ctx SpecContext) { fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { return fakeCache, nil @@ -846,8 +867,6 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).ToNot(HaveOccurred()) @@ -868,7 +887,7 @@ var _ = Describe("manger.Manager", func() { }).Should(BeTrue()) }) - It("should wait for runnables to stop", func() { + It("should wait for runnables to stop", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -902,7 +921,7 @@ var _ = Describe("manger.Manager", func() { }))).To(Succeed()) defer GinkgoRecover() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) var wgManagerRunning sync.WaitGroup wgManagerRunning.Add(1) @@ -920,7 +939,7 @@ var _ = Describe("manger.Manager", func() { wgManagerRunning.Wait() }) - It("should return an error if any Components fail to Start and wait for runnables to stop", func() { + It("should return an error if any Components fail to Start and wait for runnables to stop", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -948,13 +967,11 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() Expect(m.Start(ctx)).To(HaveOccurred()) Expect(runnableDoneCount).To(Equal(2)) }) - It("should refuse to add runnable if stop procedure is already engaged", func() { + It("should refuse to add runnable if stop procedure is already engaged", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -971,7 +988,7 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() @@ -984,7 +1001,104 @@ var _ = Describe("manger.Manager", func() { }))).NotTo(Succeed()) }) - It("should return both runnables and stop errors when both error", func() { + It("should not return runnables context.Canceled errors", func(specCtx SpecContext) { + Expect(options.Logger).To(BeZero(), "this test overrides Logger") + + var log struct { + sync.Mutex + messages []string + } + options.Logger = funcr.NewJSON(func(object string) { + log.Lock() + log.messages = append(log.messages, object) + log.Unlock() + }, funcr.Options{}) + + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + + // Runnables may return ctx.Err() as shown in some [context.Context] examples. + started := make(chan struct{}) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + close(started) + <-ctx.Done() + return ctx.Err() + }))).To(Succeed()) + + stopped := make(chan error) + ctx, cancel := context.WithCancel(specCtx) + go func() { + stopped <- m.Start(ctx) + }() + + // Wait for runnables to start, signal the manager, and wait for it to return. + <-started + cancel() + Expect(<-stopped).To(Succeed()) + + // The leader election goroutine emits one more log message after Start() returns. + // Take the lock here to avoid a race between it writing to log.messages and the + // following read from log.messages. + if options.LeaderElection { + log.Lock() + defer log.Unlock() + } + + Expect(log.messages).To(Not(ContainElement( + ContainSubstring(context.Canceled.Error()), + ))) + }) + + It("should default controller logger from manager logger", func(specCtx SpecContext) { + var lock sync.Mutex + var messages []string + options.Logger = funcr.NewJSON(func(object string) { + lock.Lock() + messages = append(messages, object) + lock.Unlock() + }, funcr.Options{}) + options.LeaderElection = false + + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + + started := make(chan struct{}) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + close(started) + return nil + }))).To(Succeed()) + + stopped := make(chan error) + ctx, cancel := context.WithCancel(specCtx) + go func() { + stopped <- m.Start(ctx) + }() + + // Wait for runnables to start as a proxy for the manager being fully started. + <-started + cancel() + Expect(<-stopped).To(Succeed()) + + msg := "controller log message" + m.GetControllerOptions().Logger.Info(msg) + + Eventually(func(g Gomega) { + lock.Lock() + defer lock.Unlock() + + g.Expect(messages).To(ContainElement( + ContainSubstring(msg), + )) + }).Should(Succeed()) + }) + + It("should return both runnables and stop errors when both error", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -1007,8 +1121,6 @@ var _ = Describe("manger.Manager", func() { return nil } }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() err = m.Start(ctx) Expect(err).To(HaveOccurred()) eMsg := "[not feeling like that, failed waiting for all runnables to end within grace period of 1ns: context deadline exceeded]" @@ -1017,7 +1129,7 @@ var _ = Describe("manger.Manager", func() { Expect(errors.Is(err, runnableError{})).To(BeTrue()) }) - It("should return only stop errors if runnables dont error", func() { + It("should return only stop errors if runnables dont error", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -1041,7 +1153,7 @@ var _ = Describe("manger.Manager", func() { return nil } }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) managerStopDone := make(chan struct{}) go func() { err = m.Start(ctx); close(managerStopDone) }() // Use the 'elected' channel to find out if startup was done, otherwise we stop @@ -1055,7 +1167,7 @@ var _ = Describe("manger.Manager", func() { Expect(errors.Is(err, runnableError{})).ToNot(BeTrue()) }) - It("should return only runnables error if stop doesn't error", func() { + It("should return only runnables error if stop doesn't error", func(ctx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -1064,8 +1176,6 @@ var _ = Describe("manger.Manager", func() { Expect(m.Add(RunnableFunc(func(context.Context) error { return runnableError{} }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() err = m.Start(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("not feeling like that")) @@ -1073,7 +1183,7 @@ var _ = Describe("manger.Manager", func() { Expect(errors.Is(err, runnableError{})).To(BeTrue()) }) - It("should not wait for runnables if gracefulShutdownTimeout is 0", func() { + It("should not wait for runnables if gracefulShutdownTimeout is 0", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -1089,7 +1199,7 @@ var _ = Describe("manger.Manager", func() { return nil }))).ToNot(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) managerStopDone := make(chan struct{}) go func() { defer GinkgoRecover() @@ -1103,7 +1213,7 @@ var _ = Describe("manger.Manager", func() { <-runnableStopped }) - It("should wait forever for runnables if gracefulShutdownTimeout is <0 (-1)", func() { + It("should wait forever for runnables if gracefulShutdownTimeout is <0 (-1)", func(specCtx SpecContext) { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -1132,7 +1242,7 @@ var _ = Describe("manger.Manager", func() { return nil }))).ToNot(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) managerStopDone := make(chan struct{}) go func() { defer GinkgoRecover() @@ -1167,42 +1277,63 @@ var _ = Describe("manger.Manager", func() { cm.onStoppedLeading = func() {} }, ) + + It("should return an error if leader election param incorrect", func(specCtx SpecContext) { + renewDeadline := time.Second * 20 + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionID: "controller-runtime", + LeaderElectionNamespace: "default", + newResourceLock: fakeleaderelection.NewResourceLock, + RenewDeadline: &renewDeadline, + }) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithTimeout(specCtx, time.Second*10) + defer cancel() + err = m.Start(ctx) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, context.DeadlineExceeded)).NotTo(BeTrue()) + }) }) Context("should start serving metrics", func() { - var listener net.Listener + var srv metricsserver.Server + var defaultServer metricsDefaultServer var opts Options BeforeEach(func() { - listener = nil + srv = nil opts = Options{ - newMetricsListener: func(addr string) (net.Listener, error) { + Metrics: metricsserver.Options{ + BindAddress: ":0", + }, + newMetricsServer: func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { var err error - listener, err = metrics.NewListener(addr) - return listener, err + srv, err = metricsserver.NewServer(options, config, httpClient) + if srv != nil { + defaultServer = srv.(metricsDefaultServer) + } + return srv, err }, } }) - AfterEach(func() { - if listener != nil { - listener.Close() - } - }) - - It("should stop serving metrics when stop is called", func() { - opts.MetricsBindAddress = ":0" + It("should stop serving metrics when stop is called", func(specCtx SpecContext) { m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) // Check the metrics started - endpoint := fmt.Sprintf("http://%s", listener.Addr().String()) + endpoint := fmt.Sprintf("http://%s/metrics", defaultServer.GetBindAddr()) _, err = http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) @@ -1213,49 +1344,49 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { _, err = http.Get(endpoint) return err - }).ShouldNot(Succeed()) + }, 10*time.Second).ShouldNot(Succeed()) }) - It("should serve metrics endpoint", func() { - opts.MetricsBindAddress = ":0" + It("should serve metrics endpoint", func(ctx SpecContext) { m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - metricsEndpoint := fmt.Sprintf("http://%s/metrics", listener.Addr().String()) + metricsEndpoint := fmt.Sprintf("http://%s/metrics", defaultServer.GetBindAddr()) resp, err := http.Get(metricsEndpoint) Expect(err).NotTo(HaveOccurred()) Expect(resp.StatusCode).To(Equal(200)) }) - It("should not serve anything other than metrics endpoint by default", func() { - opts.MetricsBindAddress = ":0" + It("should not serve anything other than metrics endpoint by default", func(ctx SpecContext) { m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - endpoint := fmt.Sprintf("http://%s/should-not-exist", listener.Addr().String()) + endpoint := fmt.Sprintf("http://%s/should-not-exist", defaultServer.GetBindAddr()) resp, err := http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(404)) }) - It("should serve metrics in its registry", func() { + It("should serve metrics in its registry", func(ctx SpecContext) { one := prometheus.NewCounter(prometheus.CounterOpts{ Name: "test_one", Help: "test metric for testing", @@ -1264,19 +1395,19 @@ var _ = Describe("manger.Manager", func() { err := metrics.Registry.Register(one) Expect(err).NotTo(HaveOccurred()) - opts.MetricsBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - metricsEndpoint := fmt.Sprintf("http://%s/metrics", listener.Addr().String()) + metricsEndpoint := fmt.Sprintf("http://%s/metrics", defaultServer.GetBindAddr()) resp, err := http.Get(metricsEndpoint) Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() @@ -1295,31 +1426,31 @@ var _ = Describe("manger.Manager", func() { Expect(ok).To(BeTrue()) }) - It("should serve extra endpoints", func() { - opts.MetricsBindAddress = ":0" + It("should serve extra endpoints", func(ctx SpecContext) { + opts.Metrics.ExtraHandlers = map[string]http.Handler{ + "/debug": http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = w.Write([]byte("Some debug info")) + }), + } m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - err = m.AddMetricsExtraHandler("/debug", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - _, _ = w.Write([]byte("Some debug info")) - })) - Expect(err).NotTo(HaveOccurred()) - // Should error when we add another extra endpoint on the already registered path. - err = m.AddMetricsExtraHandler("/debug", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + err = m.AddMetricsServerExtraHandler("/debug", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { _, _ = w.Write([]byte("Another debug info")) })) Expect(err).To(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) - endpoint := fmt.Sprintf("http://%s/debug", listener.Addr().String()) + endpoint := fmt.Sprintf("http://%s/debug", defaultServer.GetBindAddr()) resp, err := http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() @@ -1353,12 +1484,12 @@ var _ = Describe("manger.Manager", func() { } }) - It("should stop serving health probes when stop is called", func() { + It("should stop serving health probes when stop is called", func(specCtx SpecContext) { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1380,7 +1511,7 @@ var _ = Describe("manger.Manager", func() { }, 10*time.Second).ShouldNot(Succeed()) }) - It("should serve readiness endpoint", func() { + It("should serve readiness endpoint", func(ctx SpecContext) { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1390,8 +1521,6 @@ var _ = Describe("manger.Manager", func() { err = m.AddReadyzCheck(namedCheck, func(_ *http.Request) error { return res }) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1435,7 +1564,7 @@ var _ = Describe("manger.Manager", func() { Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) - It("should serve liveness endpoint", func() { + It("should serve liveness endpoint", func(ctx SpecContext) { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1445,8 +1574,6 @@ var _ = Describe("manger.Manager", func() { err = m.AddHealthzCheck(namedCheck, func(_ *http.Request) error { return res }) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1512,12 +1639,12 @@ var _ = Describe("manger.Manager", func() { } }) - It("should stop serving pprof when stop is called", func() { + It("should stop serving pprof when stop is called", func(specCtx SpecContext) { opts.PprofBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1539,13 +1666,11 @@ var _ = Describe("manger.Manager", func() { }, 10*time.Second).ShouldNot(Succeed()) }) - It("should serve pprof endpoints", func() { + It("should serve pprof endpoints", func(ctx SpecContext) { opts.PprofBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1586,7 +1711,7 @@ var _ = Describe("manger.Manager", func() { Describe("Add", func() { It("should immediately start the Component if the Manager has already Started another Component", - func() { + func(ctx SpecContext) { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) @@ -1600,8 +1725,6 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1624,14 +1747,12 @@ var _ = Describe("manger.Manager", func() { <-c2 }) - It("should immediately start the Component if the Manager has already Started", func() { + It("should immediately start the Component if the Manager has already Started", func(ctx SpecContext) { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1651,12 +1772,10 @@ var _ = Describe("manger.Manager", func() { <-c1 }) - It("should fail if attempted to start a second time", func() { + It("should fail if attempted to start a second time", func(ctx SpecContext) { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1675,13 +1794,13 @@ var _ = Describe("manger.Manager", func() { }) }) - It("should not leak goroutines when stopped", func() { + It("should not leak goroutines when stopped", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() Expect(m.Start(ctx)).NotTo(HaveOccurred()) @@ -1691,7 +1810,7 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) }) - It("should not leak goroutines if the default event broadcaster is used & events are emitted", func() { + It("should not leak goroutines if the default event broadcaster is used & events are emitted", func(specCtx SpecContext) { currentGRs := goleak.IgnoreCurrent() m, err := New(cfg, Options{ /* implicit: default setting for EventBroadcaster */ }) @@ -1708,7 +1827,7 @@ var _ = Describe("manger.Manager", func() { }))).To(Succeed()) By("starting the manager & waiting till we've sent our event") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) doneCh := make(chan struct{}) go func() { defer GinkgoRecover() @@ -1718,7 +1837,7 @@ var _ = Describe("manger.Manager", func() { <-m.Elected() Eventually(func() *corev1.Event { - evts, err := clientset.CoreV1().Events("").Search(m.GetScheme(), &ns) + evts, err := clientset.CoreV1().Events("").SearchWithContext(ctx, m.GetScheme(), &ns) Expect(err).NotTo(HaveOccurred()) for i, evt := range evts.Items { @@ -1739,6 +1858,54 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) }) + It("should not leak goroutines when a runnable returns error slowly after being signaled to stop", func(specCtx SpecContext) { + // This test reproduces the race condition where the manager's Start method + // exits due to context cancellation, leaving no one to drain errChan + + currentGRs := goleak.IgnoreCurrent() + + // Create manager with a very short graceful shutdown timeout to reliablytrigger the race condition + shortGracefulShutdownTimeout := 10 * time.Millisecond + m, err := New(cfg, Options{ + GracefulShutdownTimeout: &shortGracefulShutdownTimeout, + }) + Expect(err).NotTo(HaveOccurred()) + + // Add the slow runnable that will return an error after some delay + for i := 0; i < 3; i++ { + slowRunnable := RunnableFunc(func(c context.Context) error { + <-c.Done() + + // Simulate some work that delays the error from being returned + // Choosing a large delay to reliably trigger the race condition + time.Sleep(100 * time.Millisecond) + + // This simulates the race condition where runnables try to send + // errors after the manager has stopped reading from errChan + return errors.New("slow runnable error") + }) + + Expect(m.Add(slowRunnable)).To(Succeed()) + } + + ctx, cancel := context.WithTimeout(specCtx, 50*time.Millisecond) + defer cancel() + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).To(HaveOccurred()) // We expect error here because the slow runnables will return errors + }() + + // Wait for context to be cancelled + <-ctx.Done() + + // Give time for any leaks to become apparent. This makes sure that we don't false alarm on go routine leaks because runnables are still running. + time.Sleep(300 * time.Millisecond) + + // force-close keep-alive connections + clientTransport.CloseIdleConnections() + Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) + }) + It("should provide a function to get the Config", func() { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1781,6 +1948,76 @@ var _ = Describe("manger.Manager", func() { Expect(err).NotTo(HaveOccurred()) Expect(m.GetAPIReader()).NotTo(BeNil()) }) + + It("should run warmup runnables before leader election is won", func(ctx SpecContext) { + By("Creating a channel to track execution order") + runnableExecutionOrderChan := make(chan string, 2) + const leaderElectionRunnableName = "leaderElectionRunnable" + const warmupRunnableName = "warmupRunnable" + + By("Creating a manager with leader election enabled") + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionNamespace: "default", + LeaderElectionID: "test-leader-election-warmup", + newResourceLock: fakeleaderelection.NewResourceLock, + HealthProbeBindAddress: "0", + Metrics: metricsserver.Options{BindAddress: "0"}, + PprofBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a runnable that implements WarmupRunnable interface") + // Create a warmup runnable + warmupRunnable := newWarmupRunnableFunc( + func(ctx context.Context) error { + // This is the leader election runnable that will be executed after leader election + // It will block until context is done/cancelled + <-ctx.Done() + return nil + }, + func(ctx context.Context) error { + // This should be called during startup before leader election + runnableExecutionOrderChan <- warmupRunnableName + return nil + }, + ) + Expect(m.Add(warmupRunnable)).To(Succeed()) + + By("Creating a runnable that requires leader election") + leaderElectionRunnable := RunnableFunc( + func(ctx context.Context) error { + runnableExecutionOrderChan <- leaderElectionRunnableName + <-ctx.Done() + return nil + }, + ) + Expect(m.Add(leaderElectionRunnable)).To(Succeed()) + + cm, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + resourceLockWithHooks, ok := cm.resourceLock.(fakeleaderelection.ControllableResourceLockInterface) + Expect(ok).To(BeTrue()) + + By("Blocking leader election") + resourceLockWithHooks.BlockLeaderElection() + + By("Starting the manager") + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).To(Succeed()) + }() + + By("Waiting for the warmup runnable to be executed without leader election being won") + Expect(<-runnableExecutionOrderChan).To(Equal(warmupRunnableName)) + + By("Unblocking leader election") + resourceLockWithHooks.UnblockLeaderElection() + + By("Waiting for the leader election runnable to be executed after leader election was won") + <-m.Elected() + Expect(<-runnableExecutionOrderChan).To(Equal(leaderElectionRunnableName)) + }) }) type runnableError struct { @@ -1845,14 +2082,22 @@ func (c *startClusterAfterManager) GetCache() cache.Cache { return c.informer } -type fakeDeferredLoader struct { - *v1alpha1.ControllerManagerConfiguration +// metricsDefaultServer is used to type check the default metrics server implementation +// so we can retrieve the bind addr without having to make GetBindAddr a function on the +// metricsserver.Server interface or resort to reflection. +type metricsDefaultServer interface { + GetBindAddr() string } -func (f *fakeDeferredLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { - return f.ControllerManagerConfiguration.ControllerManagerConfigurationSpec, nil +type needElection struct { + ch chan struct{} } -func (f *fakeDeferredLoader) InjectScheme(scheme *runtime.Scheme) error { +func (n *needElection) Start(_ context.Context) error { + n.ch <- struct{}{} return nil } + +func (n *needElection) NeedLeaderElection() bool { + return true +} diff --git a/pkg/manager/runnable_group.go b/pkg/manager/runnable_group.go index 549741e6e5..53e29fc56f 100644 --- a/pkg/manager/runnable_group.go +++ b/pkg/manager/runnable_group.go @@ -5,6 +5,7 @@ import ( "errors" "sync" + "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -28,22 +29,36 @@ type runnableCheck func(ctx context.Context) bool // runnables handles all the runnables for a manager by grouping them accordingly to their // type (webhooks, caches etc.). type runnables struct { + HTTPServers *runnableGroup Webhooks *runnableGroup Caches *runnableGroup LeaderElection *runnableGroup + Warmup *runnableGroup Others *runnableGroup } // newRunnables creates a new runnables object. func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables { return &runnables{ + HTTPServers: newRunnableGroup(baseContext, errChan), Webhooks: newRunnableGroup(baseContext, errChan), Caches: newRunnableGroup(baseContext, errChan), LeaderElection: newRunnableGroup(baseContext, errChan), + Warmup: newRunnableGroup(baseContext, errChan), Others: newRunnableGroup(baseContext, errChan), } } +// withLogger returns the runnables with the logger set for all runnable groups. +func (r *runnables) withLogger(logger logr.Logger) *runnables { + r.HTTPServers.withLogger(logger) + r.Webhooks.withLogger(logger) + r.Caches.withLogger(logger) + r.LeaderElection.withLogger(logger) + r.Others.withLogger(logger) + return r +} + // Add adds a runnable to closest group of runnable that they belong to. // // Add should be able to be called before and after Start, but not after StopAndWait. @@ -52,14 +67,31 @@ func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables { // The runnables added after Start are started directly. func (r *runnables) Add(fn Runnable) error { switch runnable := fn.(type) { + case *Server: + if runnable.NeedLeaderElection() { + return r.LeaderElection.Add(fn, nil) + } + return r.HTTPServers.Add(fn, nil) case hasCache: return r.Caches.Add(fn, func(ctx context.Context) bool { return runnable.GetCache().WaitForCacheSync(ctx) }) case webhook.Server: return r.Webhooks.Add(fn, nil) - case LeaderElectionRunnable: - if !runnable.NeedLeaderElection() { + case warmupRunnable, LeaderElectionRunnable: + if warmupRunnable, ok := fn.(warmupRunnable); ok { + if err := r.Warmup.Add(RunnableFunc(warmupRunnable.Warmup), nil); err != nil { + return err + } + } + + leaderElectionRunnable, ok := fn.(LeaderElectionRunnable) + if !ok { + // If the runnable is not a LeaderElectionRunnable, add it to the leader election group for backwards compatibility + return r.LeaderElection.Add(fn, nil) + } + + if !leaderElectionRunnable.NeedLeaderElection() { return r.Others.Add(fn, nil) } return r.LeaderElection.Add(fn, nil) @@ -98,6 +130,9 @@ type runnableGroup struct { // wg is an internal sync.WaitGroup that allows us to properly stop // and wait for all the runnables to finish before returning. wg *sync.WaitGroup + + // logger is used for logging when errors are dropped during shutdown + logger logr.Logger } func newRunnableGroup(baseContext BaseContextFunc, errChan chan error) *runnableGroup { @@ -106,12 +141,18 @@ func newRunnableGroup(baseContext BaseContextFunc, errChan chan error) *runnable errChan: errChan, ch: make(chan *readyRunnable), wg: new(sync.WaitGroup), + logger: logr.Discard(), // Default to no-op logger } r.ctx, r.cancel = context.WithCancel(baseContext()) return r } +// withLogger sets the logger for this runnable group. +func (r *runnableGroup) withLogger(logger logr.Logger) { + r.logger = logger +} + // Started returns true if the group has started. func (r *runnableGroup) Started() bool { r.start.Lock() @@ -217,7 +258,27 @@ func (r *runnableGroup) reconcile() { // Start the runnable. if err := rn.Start(r.ctx); err != nil { - r.errChan <- err + // Check if we're during the shutdown process. + r.stop.RLock() + isStopped := r.stopped + r.stop.RUnlock() + + if isStopped { + // During shutdown, try to send error first (error drain goroutine might still be running) + // but drop if it would block to prevent goroutine leaks + select { + case r.errChan <- err: + // Error sent successfully (error drain goroutine is still running) + default: + // Error drain goroutine has exited, drop error to prevent goroutine leak + if !errors.Is(err, context.Canceled) { // don't log context.Canceled errors as they are expected during shutdown + r.logger.Info("error dropped during shutdown to prevent goroutine leak", "error", err) + } + } + } else { + // During normal operation, always try to send errors (may block briefly) + r.errChan <- err + } } }(runnable) } @@ -259,6 +320,15 @@ func (r *runnableGroup) Add(rn Runnable, ready runnableCheck) error { r.start.Unlock() } + // Recheck if we're stopped and hold the readlock, given that the stop and start can be called + // at the same time, we can end up in a situation where the runnable is added + // after the group is stopped and the channel is closed. + r.stop.RLock() + defer r.stop.RUnlock() + if r.stopped { + return errRunnableGroupStopped + } + // Enqueue the runnable. r.ch <- readyRunnable return nil @@ -268,7 +338,11 @@ func (r *runnableGroup) Add(rn Runnable, ready runnableCheck) error { func (r *runnableGroup) StopAndWait(ctx context.Context) { r.stopOnce.Do(func() { // Close the reconciler channel once we're done. - defer close(r.ch) + defer func() { + r.stop.Lock() + close(r.ch) + r.stop.Unlock() + }() _ = r.Start(ctx) r.stop.Lock() diff --git a/pkg/manager/runnable_group_test.go b/pkg/manager/runnable_group_test.go index 456b8d7ac0..6f9b879e0e 100644 --- a/pkg/manager/runnable_group_test.go +++ b/pkg/manager/runnable_group_test.go @@ -9,7 +9,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -21,11 +22,20 @@ var _ = Describe("runnables", func() { Expect(newRunnables(defaultBaseContext, errCh)).ToNot(BeNil()) }) + It("should add HTTP servers to the appropriate group", func() { + server := &Server{} + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(server)).To(Succeed()) + Expect(r.HTTPServers.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + It("should add caches to the appropriate group", func() { cache := &cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}} r := newRunnables(defaultBaseContext, errCh) Expect(r.Add(cache)).To(Succeed()) Expect(r.Caches.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) }) It("should add webhooks to the appropriate group", func() { @@ -33,6 +43,7 @@ var _ = Describe("runnables", func() { r := newRunnables(defaultBaseContext, errCh) Expect(r.Add(webhook)).To(Succeed()) Expect(r.Webhooks.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) }) It("should add any runnable to the leader election group", func() { @@ -44,15 +55,122 @@ var _ = Describe("runnables", func() { r := newRunnables(defaultBaseContext, errCh) Expect(r.Add(runnable)).To(Succeed()) Expect(r.LeaderElection.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should add WarmupRunnable to the Warmup and LeaderElection group", func() { + warmupRunnable := newWarmupRunnableFunc( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return nil }, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + Expect(r.Warmup.startQueue).To(HaveLen(1)) + Expect(r.LeaderElection.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should add WarmupRunnable that doesn't needs leader election to warmup group only", func() { + warmupRunnable := newLeaderElectionAndWarmupRunnable( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return nil }, + false, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + Expect(r.Warmup.startQueue).To(HaveLen(1)) + Expect(r.LeaderElection.startQueue).To(BeEmpty()) + Expect(r.Others.startQueue).To(HaveLen(1)) + }) + + It("should add WarmupRunnable that needs leader election to Warmup and LeaderElection group, not Others", func() { + warmupRunnable := newLeaderElectionAndWarmupRunnable( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return nil }, + true, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + Expect(r.Warmup.startQueue).To(HaveLen(1)) + Expect(r.LeaderElection.startQueue).To(HaveLen(1)) + Expect(r.Others.startQueue).To(BeEmpty()) + }) + + It("should execute the Warmup function when Warmup group is started", func(ctx SpecContext) { + var warmupExecuted atomic.Bool + + warmupRunnable := newWarmupRunnableFunc( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { + warmupExecuted.Store(true) + return nil + }, + ) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + // Start the Warmup group + Expect(r.Warmup.Start(ctx)).To(Succeed()) + + // Verify warmup function was called + Expect(warmupExecuted.Load()).To(BeTrue()) + }) + + It("should propagate errors from Warmup function to error channel", func(ctx SpecContext) { + expectedErr := fmt.Errorf("expected warmup error") + + warmupRunnable := newWarmupRunnableFunc( + func(c context.Context) error { + <-c.Done() + return nil + }, + func(c context.Context) error { return expectedErr }, + ) + + testErrChan := make(chan error, 1) + r := newRunnables(defaultBaseContext, testErrChan) + Expect(r.Add(warmupRunnable)).To(Succeed()) + + // Start the Warmup group in a goroutine + go func() { + Expect(r.Warmup.Start(ctx)).To(Succeed()) + }() + + // Error from Warmup should be sent to error channel + var receivedErr error + Eventually(func() error { + select { + case receivedErr = <-testErrChan: + return receivedErr + default: + return nil + } + }).Should(Equal(expectedErr)) }) }) var _ = Describe("runnableGroup", func() { errCh := make(chan error) - It("should be able to add new runnables before it starts", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should be able to add new runnables before it starts", func(ctx SpecContext) { rg := newRunnableGroup(defaultBaseContext, errCh) Expect(rg.Add(RunnableFunc(func(c context.Context) error { <-ctx.Done() @@ -62,9 +180,7 @@ var _ = Describe("runnableGroup", func() { Expect(rg.Started()).To(BeFalse()) }) - It("should be able to add new runnables before and after start", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should be able to add new runnables before and after start", func(ctx SpecContext) { rg := newRunnableGroup(defaultBaseContext, errCh) Expect(rg.Add(RunnableFunc(func(c context.Context) error { <-ctx.Done() @@ -78,9 +194,7 @@ var _ = Describe("runnableGroup", func() { }), nil)).To(Succeed()) }) - It("should be able to add new runnables before and after start concurrently", func() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + It("should be able to add new runnables before and after start concurrently", func(ctx SpecContext) { rg := newRunnableGroup(defaultBaseContext, errCh) go func() { @@ -102,10 +216,10 @@ var _ = Describe("runnableGroup", func() { } }) - It("should be able to close the group and wait for all runnables to finish", func() { - ctx, cancel := context.WithCancel(context.Background()) + It("should be able to close the group and wait for all runnables to finish", func(specCtx SpecContext) { + ctx, cancel := context.WithCancel(specCtx) - exited := pointer.Int64(0) + exited := ptr.To(int64(0)) rg := newRunnableGroup(defaultBaseContext, errCh) for i := 0; i < 10; i++ { Expect(rg.Add(RunnableFunc(func(c context.Context) error { @@ -119,7 +233,7 @@ var _ = Describe("runnableGroup", func() { // Cancel the context, asking the runnables to exit. cancel() - rg.StopAndWait(context.Background()) + rg.StopAndWait(specCtx) Expect(rg.Add(RunnableFunc(func(c context.Context) error { return nil @@ -128,8 +242,8 @@ var _ = Describe("runnableGroup", func() { Expect(atomic.LoadInt64(exited)).To(BeNumerically("==", 10)) }) - It("should be able to wait for all runnables to be ready at different intervals", func() { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + It("should be able to wait for all runnables to be ready at different intervals", func(specCtx SpecContext) { + ctx, cancel := context.WithTimeout(specCtx, 1*time.Second) defer cancel() rg := newRunnableGroup(defaultBaseContext, errCh) @@ -154,8 +268,44 @@ var _ = Describe("runnableGroup", func() { } }) - It("should not turn ready if some readiness check fail", func() { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + It("should be able to handle adding runnables while stopping", func(specCtx SpecContext) { + ctx, cancel := context.WithTimeout(specCtx, 10*time.Second) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(1 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + go func() { + defer GinkgoRecover() + <-time.After(1 * time.Millisecond) + ctx, cancel := context.WithCancel(ctx) + cancel() + rg.StopAndWait(ctx) + }() + + for i := 0; i < 200; i++ { + go func(i int) { + defer GinkgoRecover() + + <-time.After(time.Duration(i) * time.Microsecond) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), func(_ context.Context) bool { + return true + })).To(SatisfyAny( + Succeed(), + Equal(errRunnableGroupStopped), + )) + }(i) + } + }) + + It("should not turn ready if some readiness check fail", func(specCtx SpecContext) { + ctx, cancel := context.WithTimeout(specCtx, 2*time.Second) defer cancel() rg := newRunnableGroup(defaultBaseContext, errCh) @@ -180,3 +330,57 @@ var _ = Describe("runnableGroup", func() { } }) }) + +var _ warmupRunnable = &warmupRunnableFunc{} + +func newWarmupRunnableFunc( + startFunc func(context.Context) error, + warmupFunc func(context.Context) error, +) *warmupRunnableFunc { + return &warmupRunnableFunc{ + startFunc: startFunc, + warmupFunc: warmupFunc, + } +} + +// warmupRunnableFunc is a helper struct that implements WarmupRunnable +// for testing purposes. +type warmupRunnableFunc struct { + startFunc func(context.Context) error + warmupFunc func(context.Context) error +} + +func (r *warmupRunnableFunc) Start(ctx context.Context) error { + return r.startFunc(ctx) +} + +func (r *warmupRunnableFunc) Warmup(ctx context.Context) error { + return r.warmupFunc(ctx) +} + +var _ LeaderElectionRunnable = &leaderElectionAndWarmupRunnable{} +var _ warmupRunnable = &leaderElectionAndWarmupRunnable{} + +// leaderElectionAndWarmupRunnable implements both WarmupRunnable and LeaderElectionRunnable +type leaderElectionAndWarmupRunnable struct { + *warmupRunnableFunc + needLeaderElection bool +} + +func newLeaderElectionAndWarmupRunnable( + startFunc func(context.Context) error, + warmupFunc func(context.Context) error, + needLeaderElection bool, +) *leaderElectionAndWarmupRunnable { + return &leaderElectionAndWarmupRunnable{ + warmupRunnableFunc: &warmupRunnableFunc{ + startFunc: startFunc, + warmupFunc: warmupFunc, + }, + needLeaderElection: needLeaderElection, + } +} + +func (r leaderElectionAndWarmupRunnable) NeedLeaderElection() bool { + return r.needLeaderElection +} diff --git a/pkg/manager/server.go b/pkg/manager/server.go index b6509f48f2..1983165da8 100644 --- a/pkg/manager/server.go +++ b/pkg/manager/server.go @@ -21,34 +21,67 @@ import ( "errors" "net" "net/http" + "time" - "github.com/go-logr/logr" + crlog "sigs.k8s.io/controller-runtime/pkg/log" ) -// server is a general purpose HTTP server Runnable for a manager -// to serve some internal handlers such as health probes, metrics and profiling. -type server struct { - Kind string - Log logr.Logger - Server *http.Server +var ( + _ Runnable = (*Server)(nil) + _ LeaderElectionRunnable = (*Server)(nil) +) + +// Server is a general purpose HTTP server Runnable for a manager. +// It is used to serve some internal handlers for health probes and profiling, +// but it can also be used to run custom servers. +type Server struct { + // Name is an optional string that describes the purpose of the server. It is used in logs to distinguish + // among multiple servers. + Name string + + // Server is the HTTP server to run. It is required. + Server *http.Server + + // Listener is an optional listener to use. If not set, the server start a listener using the server.Addr. + // Using a listener is useful when the port reservation needs to happen in advance of this runnable starting. Listener net.Listener + + // OnlyServeWhenLeader is an optional bool that indicates that the server should only be started when the manager is the leader. + OnlyServeWhenLeader bool + + // ShutdownTimeout is an optional duration that indicates how long to wait for the server to shutdown gracefully. If not set, + // the server will wait indefinitely for all connections to close. + ShutdownTimeout *time.Duration } -func (s *server) Start(ctx context.Context) error { - log := s.Log.WithValues("kind", s.Kind, "addr", s.Listener.Addr()) +// Start starts the server. It will block until the server is stopped or an error occurs. +func (s *Server) Start(ctx context.Context) error { + log := crlog.FromContext(ctx) + if s.Name != "" { + log = log.WithValues("name", s.Name) + } + log = log.WithValues("addr", s.addr()) serverShutdown := make(chan struct{}) go func() { <-ctx.Done() log.Info("shutting down server") - if err := s.Server.Shutdown(context.Background()); err != nil { + + shutdownCtx := context.Background() + if s.ShutdownTimeout != nil { + var shutdownCancel context.CancelFunc + shutdownCtx, shutdownCancel = context.WithTimeout(shutdownCtx, *s.ShutdownTimeout) + defer shutdownCancel() + } + + if err := s.Server.Shutdown(shutdownCtx); err != nil { log.Error(err, "error shutting down server") } close(serverShutdown) }() log.Info("starting server") - if err := s.Server.Serve(s.Listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + if err := s.serve(); err != nil && !errors.Is(err, http.ErrServerClosed) { return err } @@ -56,6 +89,21 @@ func (s *server) Start(ctx context.Context) error { return nil } -func (s *server) NeedLeaderElection() bool { - return false +// NeedLeaderElection returns true if the server should only be started when the manager is the leader. +func (s *Server) NeedLeaderElection() bool { + return s.OnlyServeWhenLeader +} + +func (s *Server) addr() string { + if s.Listener != nil { + return s.Listener.Addr().String() + } + return s.Server.Addr +} + +func (s *Server) serve() error { + if s.Listener != nil { + return s.Server.Serve(s.Listener) + } + return s.Server.ListenAndServe() } diff --git a/pkg/metrics/filters/filter_suite_test.go b/pkg/metrics/filters/filter_suite_test.go new file mode 100644 index 0000000000..bdd21be491 --- /dev/null +++ b/pkg/metrics/filters/filter_suite_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "fmt" + "net/http" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +func TestSource(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Filters Suite") +} + +var testenv *envtest.Environment +var cfg *rest.Config +var clientset *kubernetes.Clientset + +// clientTransport is used to force-close keep-alives in tests that check for leaks. +var clientTransport *http.Transport + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + testenv = &envtest.Environment{} + + var err error + cfg, err = testenv.Start() + Expect(err).NotTo(HaveOccurred()) + + cfg.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { + // NB(directxman12): we can't set Transport *and* use TLS options, + // so we grab the transport right after it gets created so that we can + // type-assert on it (hopefully)? + // hopefully this doesn't break 🤞 + transport, isTransport := rt.(*http.Transport) + if !isTransport { + panic(fmt.Sprintf("wasn't able to grab underlying transport from REST client's RoundTripper, can't figure out how to close keep-alives: expected an *http.Transport, got %#v", rt)) + } + clientTransport = transport + return rt + } + + clientset, err = kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + + // Prevent the metrics listener being created + metricsserver.DefaultBindAddress = "0" +}) + +var _ = AfterSuite(func() { + Expect(testenv.Stop()).To(Succeed()) + + // Put the DefaultBindAddress back + metricsserver.DefaultBindAddress = ":8080" +}) diff --git a/pkg/metrics/filters/filters.go b/pkg/metrics/filters/filters.go new file mode 100644 index 0000000000..1659502bcf --- /dev/null +++ b/pkg/metrics/filters/filters.go @@ -0,0 +1,122 @@ +package filters + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/apiserver/pkg/authentication/authenticatorfactory" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" + "k8s.io/client-go/rest" + + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// WithAuthenticationAndAuthorization provides a metrics.Filter for authentication and authorization. +// Metrics will be authenticated (via TokenReviews) and authorized (via SubjectAccessReviews) with the +// kube-apiserver. +// For the authentication and authorization the controller needs a ClusterRole +// with the following rules: +// * apiGroups: authentication.k8s.io, resources: tokenreviews, verbs: create +// * apiGroups: authorization.k8s.io, resources: subjectaccessreviews, verbs: create +// +// To scrape metrics e.g. via Prometheus the client needs a ClusterRole +// with the following rule: +// * nonResourceURLs: "/metrics", verbs: get +// +// Note: Please note that configuring this metrics provider will introduce a dependency to "k8s.io/apiserver" +// to your go module. +func WithAuthenticationAndAuthorization(config *rest.Config, httpClient *http.Client) (metricsserver.Filter, error) { + authenticationV1Client, err := authenticationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + authorizationV1Client, err := authorizationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + + authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ + Anonymous: &apiserver.AnonymousAuthConfig{Enabled: false}, // Require authentication. + CacheTTL: 1 * time.Minute, + TokenAccessReviewClient: authenticationV1Client, + TokenAccessReviewTimeout: 10 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthenticator, _, err := authenticatorConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authenticator: %w", err) + } + + authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{ + SubjectAccessReviewClient: authorizationV1Client, + AllowCacheTTL: 5 * time.Minute, + DenyCacheTTL: 30 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthorizer, err := authorizerConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authorizer: %w", err) + } + + return func(log logr.Logger, handler http.Handler) (http.Handler, error) { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + res, ok, err := delegatingAuthenticator.AuthenticateRequest(req) + if err != nil { + log.Error(err, "Authentication failed") + http.Error(w, "Authentication failed", http.StatusInternalServerError) + return + } + if !ok { + log.V(4).Info("Authentication failed") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + attributes := authorizer.AttributesRecord{ + User: res.User, + Verb: strings.ToLower(req.Method), + Path: req.URL.Path, + } + + authorized, reason, err := delegatingAuthorizer.Authorize(ctx, attributes) + if err != nil { + msg := fmt.Sprintf("Authorization for user %s failed", attributes.User.GetName()) + log.Error(err, msg) + http.Error(w, msg, http.StatusInternalServerError) + return + } + if authorized != authorizer.DecisionAllow { + msg := fmt.Sprintf("Authorization denied for user %s", attributes.User.GetName()) + log.V(4).Info(fmt.Sprintf("%s: %s", msg, reason)) + http.Error(w, msg, http.StatusForbidden) + return + } + + handler.ServeHTTP(w, req) + }), nil + }, nil +} diff --git a/pkg/metrics/filters/filters_test.go b/pkg/metrics/filters/filters_test.go new file mode 100644 index 0000000000..bd107fc56d --- /dev/null +++ b/pkg/metrics/filters/filters_test.go @@ -0,0 +1,271 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filters + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net/http" + "reflect" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +var _ = Describe("manger.Manager", func() { + Describe("Start", func() { + Context("should start serving metrics with https and authn/authz", func() { + var srv metricsserver.Server + var defaultServer metricsDefaultServer + var opts manager.Options + var httpClient *http.Client + + BeforeEach(func() { + srv = nil + newMetricsServer := func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) { + var err error + srv, err = metricsserver.NewServer(options, config, httpClient) + if srv != nil { + defaultServer = srv.(metricsDefaultServer) + } + return srv, err + } + opts = manager.Options{ + Metrics: metricsserver.Options{ + BindAddress: ":0", + SecureServing: true, + FilterProvider: WithAuthenticationAndAuthorization, + }, + } + v := reflect.ValueOf(&opts).Elem() + newMetricsField := v.FieldByName("newMetricsServer") + reflect.NewAt(newMetricsField.Type(), newMetricsField.Addr().UnsafePointer()). + Elem(). + Set(reflect.ValueOf(newMetricsServer)) + httpClient = &http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }} + }) + + It("should serve metrics in its registry", func(ctx SpecContext) { + one := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "test_one", + Help: "test metric for testing", + }) + one.Inc() + err := metrics.Registry.Register(one) + Expect(err).NotTo(HaveOccurred()) + + m, err := manager.New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) + + // Setup service account with rights to "/metrics" + token, cleanup, err := setupServiceAccountForURL(ctx, m.GetClient(), "/metrics") + defer cleanup() + Expect(err).ToNot(HaveOccurred()) + + // GET /metrics with token. + metricsEndpoint := fmt.Sprintf("https://%s/metrics", defaultServer.GetBindAddr()) + req, err := http.NewRequest("GET", metricsEndpoint, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err := httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + // This is expected as the token has rights for /metrics. + Expect(resp.StatusCode).To(Equal(200)) + + data, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring("%s\n%s\n%s\n", + `# HELP test_one test metric for testing`, + `# TYPE test_one counter`, + `test_one 1`, + )) + + // Unregister will return false if the metric was never registered + ok := metrics.Registry.Unregister(one) + Expect(ok).To(BeTrue()) + }) + + It("should serve extra endpoints", func(ctx SpecContext) { + opts.Metrics.ExtraHandlers = map[string]http.Handler{ + "/debug": http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = w.Write([]byte("Some debug info")) + }), + } + m, err := manager.New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + <-m.Elected() + // Note: Wait until metrics server has been started. A finished leader election + // doesn't guarantee that the metrics server is up. + Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty()) + + // Setup service account with rights to "/debug" + token, cleanup, err := setupServiceAccountForURL(ctx, m.GetClient(), "/debug") + defer cleanup() + Expect(err).ToNot(HaveOccurred()) + + // GET /debug without token. + endpoint := fmt.Sprintf("https://%s/debug", defaultServer.GetBindAddr()) + req, err := http.NewRequest("GET", endpoint, nil) + Expect(err).NotTo(HaveOccurred()) + resp, err := httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + // This is expected as we didn't send a token. + Expect(resp.StatusCode).To(Equal(401)) + body, err := io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(body)).To(ContainSubstring("Unauthorized")) + + // PUT /debug with token. + req, err = http.NewRequest("PUT", endpoint, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err = httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + // This is expected as the token has rights for /debug. + Expect(resp.StatusCode).To(Equal(200)) + body, err = io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + Expect(string(body)).To(Equal("Some debug info")) + + // GET /metrics with token (but token only has rights for /debug). + metricsEndpoint := fmt.Sprintf("https://%s/metrics", defaultServer.GetBindAddr()) + req, err = http.NewRequest("GET", metricsEndpoint, nil) + Expect(err).NotTo(HaveOccurred()) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + resp, err = httpClient.Do(req) + Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(403)) + body, err = io.ReadAll(resp.Body) + Expect(err).NotTo(HaveOccurred()) + // Authorization denied is expected as the token only has rights for /debug not for /metrics. + Expect(string(body)).To(ContainSubstring("Authorization denied for user system:serviceaccount:default:metrics-test")) + }) + }) + }) +}) + +type metricsDefaultServer interface { + GetBindAddr() string +} + +func setupServiceAccountForURL(ctx context.Context, c client.Client, path string) (string, func(), error) { + createdObjects := []client.Object{} + cleanup := func() { + for _, obj := range createdObjects { + _ = c.Delete(ctx, obj) + } + } + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + Namespace: metav1.NamespaceDefault, + }, + } + if err := c.Create(ctx, sa); err != nil { + return "", cleanup, err + } + createdObjects = append(createdObjects, sa) + + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + }, + Rules: []rbacv1.PolicyRule{ + { + Verbs: []string{"get", "put"}, + NonResourceURLs: []string{path}, + }, + }, + } + if err := c.Create(ctx, cr); err != nil { + return "", cleanup, err + } + createdObjects = append(createdObjects, cr) + + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-test", + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "metrics-test", + Namespace: metav1.NamespaceDefault, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: "metrics-test", + }, + } + if err := c.Create(ctx, crb); err != nil { + return "", cleanup, err + } + createdObjects = append(createdObjects, crb) + + tokenRequest := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: ptr.To(int64(2 * 60 * 60)), // 2 hours. + }, + } + if err := c.SubResource("token").Create(ctx, sa, tokenRequest); err != nil { + return "", cleanup, err + } + + if tokenRequest.Status.Token == "" { + return "", cleanup, errors.New("failed to get ServiceAccount token: token should not be empty") + } + + return tokenRequest.Status.Token, cleanup, nil +} diff --git a/pkg/metrics/leaderelection.go b/pkg/metrics/leaderelection.go index a19c099602..61e1009d32 100644 --- a/pkg/metrics/leaderelection.go +++ b/pkg/metrics/leaderelection.go @@ -14,6 +14,11 @@ var ( Name: "leader_election_master_status", Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.", }, []string{"name"}) + + leaderSlowpathCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "leader_election_slowpath_total", + Help: "Total number of slow path exercised in renewing leader leases. 'name' is the string used to identify the lease. Please make sure to group by name.", + }, []string{"name"}) ) func init() { @@ -23,18 +28,20 @@ func init() { type leaderelectionMetricsProvider struct{} -func (leaderelectionMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric { - return &switchAdapter{gauge: leaderGauge} +func (leaderelectionMetricsProvider) NewLeaderMetric() leaderelection.LeaderMetric { + return leaderElectionPrometheusAdapter{} } -type switchAdapter struct { - gauge *prometheus.GaugeVec +type leaderElectionPrometheusAdapter struct{} + +func (s leaderElectionPrometheusAdapter) On(name string) { + leaderGauge.WithLabelValues(name).Set(1.0) } -func (s *switchAdapter) On(name string) { - s.gauge.WithLabelValues(name).Set(1.0) +func (s leaderElectionPrometheusAdapter) Off(name string) { + leaderGauge.WithLabelValues(name).Set(0.0) } -func (s *switchAdapter) Off(name string) { - s.gauge.WithLabelValues(name).Set(0.0) +func (leaderElectionPrometheusAdapter) SlowpathExercised(name string) { + leaderSlowpathCounter.WithLabelValues(name).Inc() } diff --git a/pkg/metrics/listener.go b/pkg/metrics/listener.go deleted file mode 100644 index 123d8c15f9..0000000000 --- a/pkg/metrics/listener.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "fmt" - "net" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" -) - -var log = logf.RuntimeLog.WithName("metrics") - -// DefaultBindAddress sets the default bind address for the metrics listener -// The metrics is on by default. -var DefaultBindAddress = ":8080" - -// NewListener creates a new TCP listener bound to the given address. -func NewListener(addr string) (net.Listener, error) { - if addr == "" { - // If the metrics bind address is empty, default to ":8080" - addr = DefaultBindAddress - } - - // Add a case to disable metrics altogether - if addr == "0" { - return nil, nil - } - - log.Info("Metrics server is starting to listen", "addr", addr) - ln, err := net.Listen("tcp", addr) - if err != nil { - er := fmt.Errorf("error listening on %s: %w", addr, err) - log.Error(er, "metrics server failed to listen. You may want to disable the metrics server or use another port if it is due to conflicts") - return nil, er - } - return ln, nil -} diff --git a/pkg/config/doc.go b/pkg/metrics/server/doc.go similarity index 69% rename from pkg/config/doc.go rename to pkg/metrics/server/doc.go index 47a5a2f1d7..4c42f6eed7 100644 --- a/pkg/config/doc.go +++ b/pkg/metrics/server/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Kubernetes Authors. +Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,6 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package config contains functionality for interacting with -// configuration for controller-runtime components. -package config +/* +Package server provides the metrics server implementation. +*/ +package server + +import ( + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" +) + +var log = logf.RuntimeLog.WithName("metrics") diff --git a/pkg/metrics/server/server.go b/pkg/metrics/server/server.go new file mode 100644 index 0000000000..939c333f7a --- /dev/null +++ b/pkg/metrics/server/server.go @@ -0,0 +1,340 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/client-go/rest" + certutil "k8s.io/client-go/util/cert" + + "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + defaultMetricsEndpoint = "/metrics" +) + +// DefaultBindAddress is the default bind address for the metrics server. +var DefaultBindAddress = ":8080" + +// Server is a server that serves metrics. +type Server interface { + // AddExtraHandler adds extra handler served on path to the http server that serves metrics. + AddExtraHandler(path string, handler http.Handler) error + + // NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates + // the metrics server doesn't need leader election. + NeedLeaderElection() bool + + // Start runs the server. + // It will install the metrics related resources depending on the server configuration. + Start(ctx context.Context) error +} + +// Options are all available options for the metrics.Server +type Options struct { + // SecureServing enables serving metrics via https. + // Per default metrics will be served via http. + SecureServing bool + + // BindAddress is the bind address for the metrics server. + // It will be defaulted to ":8080" if unspecified. + // Set this to "0" to disable the metrics server. + BindAddress string + + // ExtraHandlers contains a map of handlers (by path) which will be added to the metrics server. + // This might be useful to register diagnostic endpoints e.g. pprof. + // Note that pprof endpoints are meant to be sensitive and shouldn't be exposed publicly. + // If the simple path -> handler mapping offered here is not enough, a new http + // server/listener should be added as Runnable to the manager via the Add method. + ExtraHandlers map[string]http.Handler + + // FilterProvider provides a filter which is a func that is added around + // the metrics and the extra handlers on the metrics server. + // This can be e.g. used to enforce authentication and authorization on the handlers + // endpoint by setting this field to filters.WithAuthenticationAndAuthorization. + FilterProvider func(c *rest.Config, httpClient *http.Client) (Filter, error) + + // CertDir is the directory that contains the server key and certificate. Defaults to + // /k8s-metrics-server/serving-certs. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. + // Note: If certificate or key doesn't exist a self-signed certificate will be used. + CertDir string + + // CertName is the server certificate name. Defaults to tls.crt. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. + // Note: If certificate or key doesn't exist a self-signed certificate will be used. + CertName string + + // KeyName is the server key name. Defaults to tls.key. + // + // Note: This option is only used when TLSOpts does not set GetCertificate. + // Note: If certificate or key doesn't exist a self-signed certificate will be used. + KeyName string + + // TLSOpts is used to allow configuring the TLS config used for the server. + // This also allows providing a certificate via GetCertificate. + TLSOpts []func(*tls.Config) + + // ListenConfig contains options for listening to an address on the metric server. + ListenConfig net.ListenConfig +} + +// Filter is a func that is added around metrics and extra handlers on the metrics server. +type Filter func(log logr.Logger, handler http.Handler) (http.Handler, error) + +// NewServer constructs a new metrics.Server from the provided options. +func NewServer(o Options, config *rest.Config, httpClient *http.Client) (Server, error) { + o.setDefaults() + + // Skip server creation if metrics are disabled. + if o.BindAddress == "0" { + return nil, nil + } + + // Validate that ExtraHandlers is not overwriting the default /metrics endpoint. + if o.ExtraHandlers != nil { + if _, ok := o.ExtraHandlers[defaultMetricsEndpoint]; ok { + return nil, fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint) + } + } + + // Create the metrics filter if a FilterProvider is set. + var metricsFilter Filter + if o.FilterProvider != nil { + var err error + metricsFilter, err = o.FilterProvider(config, httpClient) + if err != nil { + return nil, fmt.Errorf("filter provider failed to create filter for the metrics server: %w", err) + } + } + + return &defaultServer{ + metricsFilter: metricsFilter, + options: o, + }, nil +} + +// defaultServer is the default implementation used for Server. +type defaultServer struct { + options Options + + // metricsFilter is a filter which is added around + // the metrics and the extra handlers on the metrics server. + metricsFilter Filter + + // mu protects access to the bindAddr field. + mu sync.RWMutex + + // bindAddr is used to store the bindAddr after the listener has been created. + // This is used during testing to figure out the port that has been chosen randomly. + bindAddr string +} + +// setDefaults does defaulting for the Server. +func (o *Options) setDefaults() { + if o.BindAddress == "" { + o.BindAddress = DefaultBindAddress + } + + if len(o.CertDir) == 0 { + o.CertDir = filepath.Join(os.TempDir(), "k8s-metrics-server", "serving-certs") + } + + if len(o.CertName) == 0 { + o.CertName = "tls.crt" + } + + if len(o.KeyName) == 0 { + o.KeyName = "tls.key" + } +} + +// NeedLeaderElection implements the LeaderElectionRunnable interface, which indicates +// the metrics server doesn't need leader election. +func (*defaultServer) NeedLeaderElection() bool { + return false +} + +// AddExtraHandler adds extra handler served on path to the http server that serves metrics. +func (s *defaultServer) AddExtraHandler(path string, handler http.Handler) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.options.ExtraHandlers == nil { + s.options.ExtraHandlers = make(map[string]http.Handler) + } + if path == defaultMetricsEndpoint { + return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint) + } + if _, found := s.options.ExtraHandlers[path]; found { + return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path) + } + s.options.ExtraHandlers[path] = handler + return nil +} + +// Start runs the server. +// It will install the metrics related resources depend on the server configuration. +func (s *defaultServer) Start(ctx context.Context) error { + log.Info("Starting metrics server") + + listener, err := s.createListener(ctx, log) + if err != nil { + return fmt.Errorf("failed to start metrics server: failed to create listener: %w", err) + } + // Storing bindAddr here so we can retrieve it during testing via GetBindAddr. + s.mu.Lock() + s.bindAddr = listener.Addr().String() + s.mu.Unlock() + + mux := http.NewServeMux() + + handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{ + ErrorHandling: promhttp.HTTPErrorOnError, + }) + if s.metricsFilter != nil { + log := log.WithValues("path", defaultMetricsEndpoint) + var err error + handler, err = s.metricsFilter(log, handler) + if err != nil { + return fmt.Errorf("failed to start metrics server: failed to add metrics filter: %w", err) + } + } + // TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics + mux.Handle(defaultMetricsEndpoint, handler) + + for path, extraHandler := range s.options.ExtraHandlers { + if s.metricsFilter != nil { + log := log.WithValues("path", path) + var err error + extraHandler, err = s.metricsFilter(log, extraHandler) + if err != nil { + return fmt.Errorf("failed to start metrics server: failed to add metrics filter to extra handler for path %s: %w", path, err) + } + } + mux.Handle(path, extraHandler) + } + + log.Info("Serving metrics server", "bindAddress", s.options.BindAddress, "secure", s.options.SecureServing) + + srv := httpserver.New(mux) + + idleConnsClosed := make(chan struct{}) + go func() { + <-ctx.Done() + log.Info("Shutting down metrics server with timeout of 1 minute") + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + // Error from closing listeners, or context timeout + log.Error(err, "error shutting down the HTTP server") + } + close(idleConnsClosed) + }() + + if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed { + return err + } + + <-idleConnsClosed + return nil +} + +func (s *defaultServer) createListener(ctx context.Context, log logr.Logger) (net.Listener, error) { + if !s.options.SecureServing { + return s.options.ListenConfig.Listen(ctx, "tcp", s.options.BindAddress) + } + + cfg := &tls.Config{ + NextProtos: []string{"h2"}, + } + // fallback TLS config ready, will now mutate if passer wants full control over it + for _, op := range s.options.TLSOpts { + op(cfg) + } + + if cfg.GetCertificate == nil { + certPath := filepath.Join(s.options.CertDir, s.options.CertName) + keyPath := filepath.Join(s.options.CertDir, s.options.KeyName) + + _, certErr := os.Stat(certPath) + certExists := !os.IsNotExist(certErr) + _, keyErr := os.Stat(keyPath) + keyExists := !os.IsNotExist(keyErr) + if certExists && keyExists { + // Create the certificate watcher and + // set the config's GetCertificate on the TLSConfig + certWatcher, err := certwatcher.New(certPath, keyPath) + if err != nil { + return nil, err + } + cfg.GetCertificate = certWatcher.GetCertificate + + go func() { + if err := certWatcher.Start(ctx); err != nil { + log.Error(err, "certificate watcher error") + } + }() + } + } + + // If cfg.GetCertificate is still nil, i.e. we didn't configure a cert watcher, fallback to a self-signed certificate. + if cfg.GetCertificate == nil { + // Note: Using self-signed certificates here should be good enough. It's just important that we + // encrypt the communication. For example kube-controller-manager also uses a self-signed certificate + // for the metrics endpoint per default. + cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures("localhost", []net.IP{{127, 0, 0, 1}}, nil, "") + if err != nil { + return nil, fmt.Errorf("failed to generate self-signed certificate for metrics server: %w", err) + } + + keyPair, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, fmt.Errorf("failed to create self-signed key pair for metrics server: %w", err) + } + cfg.Certificates = []tls.Certificate{keyPair} + } + + l, err := s.options.ListenConfig.Listen(ctx, "tcp", s.options.BindAddress) + if err != nil { + return nil, err + } + + return tls.NewListener(l, cfg), nil +} + +func (s *defaultServer) GetBindAddr() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.bindAddr +} diff --git a/pkg/metrics/workqueue.go b/pkg/metrics/workqueue.go index 277b878810..cd7ccc773e 100644 --- a/pkg/metrics/workqueue.go +++ b/pkg/metrics/workqueue.go @@ -16,15 +16,6 @@ limitations under the License. package metrics -import ( - "github.com/prometheus/client_golang/prometheus" - "k8s.io/client-go/util/workqueue" -) - -// This file is copied and adapted from k8s.io/component-base/metrics/prometheus/workqueue -// which registers metrics to the k8s legacy Registry. We require very -// similar functionality, but must register metrics to a different Registry. - // Metrics subsystem and all keys used by the workqueue. const ( WorkQueueSubsystem = "workqueue" @@ -36,95 +27,3 @@ const ( LongestRunningProcessorKey = "longest_running_processor_seconds" RetriesKey = "retries_total" ) - -var ( - depth = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: DepthKey, - Help: "Current depth of workqueue", - }, []string{"name"}) - - adds = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: WorkQueueSubsystem, - Name: AddsKey, - Help: "Total number of adds handled by workqueue", - }, []string{"name"}) - - latency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: WorkQueueSubsystem, - Name: QueueLatencyKey, - Help: "How long in seconds an item stays in workqueue before being requested", - Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), - }, []string{"name"}) - - workDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ - Subsystem: WorkQueueSubsystem, - Name: WorkDurationKey, - Help: "How long in seconds processing an item from workqueue takes.", - Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), - }, []string{"name"}) - - unfinished = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: UnfinishedWorkKey, - Help: "How many seconds of work has been done that " + - "is in progress and hasn't been observed by work_duration. Large " + - "values indicate stuck threads. One can deduce the number of stuck " + - "threads by observing the rate at which this increases.", - }, []string{"name"}) - - longestRunningProcessor = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: WorkQueueSubsystem, - Name: LongestRunningProcessorKey, - Help: "How many seconds has the longest running " + - "processor for workqueue been running.", - }, []string{"name"}) - - retries = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: WorkQueueSubsystem, - Name: RetriesKey, - Help: "Total number of retries handled by workqueue", - }, []string{"name"}) -) - -func init() { - Registry.MustRegister(depth) - Registry.MustRegister(adds) - Registry.MustRegister(latency) - Registry.MustRegister(workDuration) - Registry.MustRegister(unfinished) - Registry.MustRegister(longestRunningProcessor) - Registry.MustRegister(retries) - - workqueue.SetProvider(workqueueMetricsProvider{}) -} - -type workqueueMetricsProvider struct{} - -func (workqueueMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric { - return depth.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric { - return adds.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric { - return latency.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric { - return workDuration.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric { - return unfinished.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric { - return longestRunningProcessor.WithLabelValues(name) -} - -func (workqueueMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric { - return retries.WithLabelValues(name) -} diff --git a/pkg/predicate/predicate.go b/pkg/predicate/predicate.go index 314635875e..9f24cb178c 100644 --- a/pkg/predicate/predicate.go +++ b/pkg/predicate/predicate.go @@ -17,6 +17,7 @@ limitations under the License. package predicate import ( + "maps" "reflect" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,45 +30,53 @@ import ( var log = logf.RuntimeLog.WithName("predicate").WithName("eventFilters") // Predicate filters events before enqueuing the keys. -type Predicate interface { +type Predicate = TypedPredicate[client.Object] + +// TypedPredicate filters events before enqueuing the keys. +type TypedPredicate[object any] interface { // Create returns true if the Create event should be processed - Create(event.CreateEvent) bool + Create(event.TypedCreateEvent[object]) bool // Delete returns true if the Delete event should be processed - Delete(event.DeleteEvent) bool + Delete(event.TypedDeleteEvent[object]) bool // Update returns true if the Update event should be processed - Update(event.UpdateEvent) bool + Update(event.TypedUpdateEvent[object]) bool // Generic returns true if the Generic event should be processed - Generic(event.GenericEvent) bool + Generic(event.TypedGenericEvent[object]) bool } -var _ Predicate = Funcs{} -var _ Predicate = ResourceVersionChangedPredicate{} -var _ Predicate = GenerationChangedPredicate{} -var _ Predicate = AnnotationChangedPredicate{} -var _ Predicate = or{} -var _ Predicate = and{} -var _ Predicate = not{} +var ( + _ Predicate = Funcs{} + _ Predicate = ResourceVersionChangedPredicate{} + _ Predicate = GenerationChangedPredicate{} + _ Predicate = AnnotationChangedPredicate{} + _ Predicate = or[client.Object]{} + _ Predicate = and[client.Object]{} + _ Predicate = not[client.Object]{} +) // Funcs is a function that implements Predicate. -type Funcs struct { +type Funcs = TypedFuncs[client.Object] + +// TypedFuncs is a function that implements TypedPredicate. +type TypedFuncs[object any] struct { // Create returns true if the Create event should be processed - CreateFunc func(event.CreateEvent) bool + CreateFunc func(event.TypedCreateEvent[object]) bool // Delete returns true if the Delete event should be processed - DeleteFunc func(event.DeleteEvent) bool + DeleteFunc func(event.TypedDeleteEvent[object]) bool // Update returns true if the Update event should be processed - UpdateFunc func(event.UpdateEvent) bool + UpdateFunc func(event.TypedUpdateEvent[object]) bool // Generic returns true if the Generic event should be processed - GenericFunc func(event.GenericEvent) bool + GenericFunc func(event.TypedGenericEvent[object]) bool } // Create implements Predicate. -func (p Funcs) Create(e event.CreateEvent) bool { +func (p TypedFuncs[object]) Create(e event.TypedCreateEvent[object]) bool { if p.CreateFunc != nil { return p.CreateFunc(e) } @@ -75,7 +84,7 @@ func (p Funcs) Create(e event.CreateEvent) bool { } // Delete implements Predicate. -func (p Funcs) Delete(e event.DeleteEvent) bool { +func (p TypedFuncs[object]) Delete(e event.TypedDeleteEvent[object]) bool { if p.DeleteFunc != nil { return p.DeleteFunc(e) } @@ -83,7 +92,7 @@ func (p Funcs) Delete(e event.DeleteEvent) bool { } // Update implements Predicate. -func (p Funcs) Update(e event.UpdateEvent) bool { +func (p TypedFuncs[object]) Update(e event.TypedUpdateEvent[object]) bool { if p.UpdateFunc != nil { return p.UpdateFunc(e) } @@ -91,7 +100,7 @@ func (p Funcs) Update(e event.UpdateEvent) bool { } // Generic implements Predicate. -func (p Funcs) Generic(e event.GenericEvent) bool { +func (p TypedFuncs[object]) Generic(e event.TypedGenericEvent[object]) bool { if p.GenericFunc != nil { return p.GenericFunc(e) } @@ -118,18 +127,41 @@ func NewPredicateFuncs(filter func(object client.Object) bool) Funcs { } } +// NewTypedPredicateFuncs returns a predicate funcs that applies the given filter function +// on CREATE, UPDATE, DELETE and GENERIC events. For UPDATE events, the filter is applied +// to the new object. +func NewTypedPredicateFuncs[object any](filter func(object object) bool) TypedFuncs[object] { + return TypedFuncs[object]{ + CreateFunc: func(e event.TypedCreateEvent[object]) bool { + return filter(e.Object) + }, + UpdateFunc: func(e event.TypedUpdateEvent[object]) bool { + return filter(e.ObjectNew) + }, + DeleteFunc: func(e event.TypedDeleteEvent[object]) bool { + return filter(e.Object) + }, + GenericFunc: func(e event.TypedGenericEvent[object]) bool { + return filter(e.Object) + }, + } +} + // ResourceVersionChangedPredicate implements a default update predicate function on resource version change. -type ResourceVersionChangedPredicate struct { - Funcs +type ResourceVersionChangedPredicate = TypedResourceVersionChangedPredicate[client.Object] + +// TypedResourceVersionChangedPredicate implements a default update predicate function on resource version change. +type TypedResourceVersionChangedPredicate[T metav1.Object] struct { + TypedFuncs[T] } // Update implements default UpdateEvent filter for validating resource version change. -func (ResourceVersionChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedResourceVersionChangedPredicate[T]) Update(e event.TypedUpdateEvent[T]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object to update", "event", e) return false } @@ -143,7 +175,27 @@ func (ResourceVersionChangedPredicate) Update(e event.UpdateEvent) bool { // The metadata.generation field of an object is incremented by the API server when writes are made to the spec field of an object. // This allows a controller to ignore update events where the spec is unchanged, and only the metadata and/or status fields are changed. // -// For CustomResource objects the Generation is only incremented when the status subresource is enabled. +// For CustomResource objects the Generation is incremented when spec is changed, or status changed and status not modeled as subresource. +// subresource status update will not increase Generation. +// +// Caveats: +// +// * The assumption that the Generation is incremented only on writing to the spec does not hold for all APIs. +// E.g For Deployment objects the Generation is also incremented on writes to the metadata.annotations field. +// For object types other than CustomResources be sure to verify which fields will trigger a Generation increment when they are written to. +// +// * With this predicate, any update events with writes only to the status field will not be reconciled. +// So in the event that the status block is overwritten or wiped by someone else the controller will not self-correct to restore the correct status. +type GenerationChangedPredicate = TypedGenerationChangedPredicate[client.Object] + +// TypedGenerationChangedPredicate implements a default update predicate function on Generation change. +// +// This predicate will skip update events that have no change in the object's metadata.generation field. +// The metadata.generation field of an object is incremented by the API server when writes are made to the spec field of an object. +// This allows a controller to ignore update events where the spec is unchanged, and only the metadata and/or status fields are changed. +// +// For CustomResource objects the Generation is incremented when spec is changed, or status changed and status not modeled as subresource. +// subresource status update will not increase Generation. // // Caveats: // @@ -153,17 +205,17 @@ func (ResourceVersionChangedPredicate) Update(e event.UpdateEvent) bool { // // * With this predicate, any update events with writes only to the status field will not be reconciled. // So in the event that the status block is overwritten or wiped by someone else the controller will not self-correct to restore the correct status. -type GenerationChangedPredicate struct { - Funcs +type TypedGenerationChangedPredicate[object metav1.Object] struct { + TypedFuncs[object] } // Update implements default UpdateEvent filter for validating generation change. -func (GenerationChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedGenerationChangedPredicate[object]) Update(e event.TypedUpdateEvent[object]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object for update", "event", e) return false } @@ -183,22 +235,25 @@ func (GenerationChangedPredicate) Update(e event.UpdateEvent) bool { // // This is mostly useful for controllers that needs to trigger both when the resource's generation is incremented // (i.e., when the resource' .spec changes), or an annotation changes (e.g., for a staging/alpha API). -type AnnotationChangedPredicate struct { - Funcs +type AnnotationChangedPredicate = TypedAnnotationChangedPredicate[client.Object] + +// TypedAnnotationChangedPredicate implements a default update predicate function on annotation change. +type TypedAnnotationChangedPredicate[object metav1.Object] struct { + TypedFuncs[object] } // Update implements default UpdateEvent filter for validating annotation change. -func (AnnotationChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedAnnotationChangedPredicate[object]) Update(e event.TypedUpdateEvent[object]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object for update", "event", e) return false } - return !reflect.DeepEqual(e.ObjectNew.GetAnnotations(), e.ObjectOld.GetAnnotations()) + return !maps.Equal(e.ObjectNew.GetAnnotations(), e.ObjectOld.GetAnnotations()) } // LabelChangedPredicate implements a default update predicate function on label change. @@ -206,42 +261,44 @@ func (AnnotationChangedPredicate) Update(e event.UpdateEvent) bool { // This predicate will skip update events that have no change in the object's label. // It is intended to be used in conjunction with the GenerationChangedPredicate, as in the following example: // -// Controller.Watch( -// -// &source.Kind{Type: v1.MyCustomKind}, -// &handler.EnqueueRequestForObject{}, -// predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})) +// Controller.Watch( +// &source.Kind{Type: v1.MyCustomKind}, +// &handler.EnqueueRequestForObject{}, +// predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})) // // This will be helpful when object's labels is carrying some extra specification information beyond object's spec, // and the controller will be triggered if any valid spec change (not only in spec, but also in labels) happens. -type LabelChangedPredicate struct { - Funcs +type LabelChangedPredicate = TypedLabelChangedPredicate[client.Object] + +// TypedLabelChangedPredicate implements a default update predicate function on label change. +type TypedLabelChangedPredicate[object metav1.Object] struct { + TypedFuncs[object] } // Update implements default UpdateEvent filter for checking label change. -func (LabelChangedPredicate) Update(e event.UpdateEvent) bool { - if e.ObjectOld == nil { +func (TypedLabelChangedPredicate[object]) Update(e event.TypedUpdateEvent[object]) bool { + if isNil(e.ObjectOld) { log.Error(nil, "Update event has no old object to update", "event", e) return false } - if e.ObjectNew == nil { + if isNil(e.ObjectNew) { log.Error(nil, "Update event has no new object for update", "event", e) return false } - return !reflect.DeepEqual(e.ObjectNew.GetLabels(), e.ObjectOld.GetLabels()) + return !maps.Equal(e.ObjectNew.GetLabels(), e.ObjectOld.GetLabels()) } // And returns a composite predicate that implements a logical AND of the predicates passed to it. -func And(predicates ...Predicate) Predicate { - return and{predicates} +func And[object any](predicates ...TypedPredicate[object]) TypedPredicate[object] { + return and[object]{predicates} } -type and struct { - predicates []Predicate +type and[object any] struct { + predicates []TypedPredicate[object] } -func (a and) Create(e event.CreateEvent) bool { +func (a and[object]) Create(e event.TypedCreateEvent[object]) bool { for _, p := range a.predicates { if !p.Create(e) { return false @@ -250,7 +307,7 @@ func (a and) Create(e event.CreateEvent) bool { return true } -func (a and) Update(e event.UpdateEvent) bool { +func (a and[object]) Update(e event.TypedUpdateEvent[object]) bool { for _, p := range a.predicates { if !p.Update(e) { return false @@ -259,7 +316,7 @@ func (a and) Update(e event.UpdateEvent) bool { return true } -func (a and) Delete(e event.DeleteEvent) bool { +func (a and[object]) Delete(e event.TypedDeleteEvent[object]) bool { for _, p := range a.predicates { if !p.Delete(e) { return false @@ -268,7 +325,7 @@ func (a and) Delete(e event.DeleteEvent) bool { return true } -func (a and) Generic(e event.GenericEvent) bool { +func (a and[object]) Generic(e event.TypedGenericEvent[object]) bool { for _, p := range a.predicates { if !p.Generic(e) { return false @@ -278,15 +335,15 @@ func (a and) Generic(e event.GenericEvent) bool { } // Or returns a composite predicate that implements a logical OR of the predicates passed to it. -func Or(predicates ...Predicate) Predicate { - return or{predicates} +func Or[object any](predicates ...TypedPredicate[object]) TypedPredicate[object] { + return or[object]{predicates} } -type or struct { - predicates []Predicate +type or[object any] struct { + predicates []TypedPredicate[object] } -func (o or) Create(e event.CreateEvent) bool { +func (o or[object]) Create(e event.TypedCreateEvent[object]) bool { for _, p := range o.predicates { if p.Create(e) { return true @@ -295,7 +352,7 @@ func (o or) Create(e event.CreateEvent) bool { return false } -func (o or) Update(e event.UpdateEvent) bool { +func (o or[object]) Update(e event.TypedUpdateEvent[object]) bool { for _, p := range o.predicates { if p.Update(e) { return true @@ -304,7 +361,7 @@ func (o or) Update(e event.UpdateEvent) bool { return false } -func (o or) Delete(e event.DeleteEvent) bool { +func (o or[object]) Delete(e event.TypedDeleteEvent[object]) bool { for _, p := range o.predicates { if p.Delete(e) { return true @@ -313,7 +370,7 @@ func (o or) Delete(e event.DeleteEvent) bool { return false } -func (o or) Generic(e event.GenericEvent) bool { +func (o or[object]) Generic(e event.TypedGenericEvent[object]) bool { for _, p := range o.predicates { if p.Generic(e) { return true @@ -323,27 +380,27 @@ func (o or) Generic(e event.GenericEvent) bool { } // Not returns a predicate that implements a logical NOT of the predicate passed to it. -func Not(predicate Predicate) Predicate { - return not{predicate} +func Not[object any](predicate TypedPredicate[object]) TypedPredicate[object] { + return not[object]{predicate} } -type not struct { - predicate Predicate +type not[object any] struct { + predicate TypedPredicate[object] } -func (n not) Create(e event.CreateEvent) bool { +func (n not[object]) Create(e event.TypedCreateEvent[object]) bool { return !n.predicate.Create(e) } -func (n not) Update(e event.UpdateEvent) bool { +func (n not[object]) Update(e event.TypedUpdateEvent[object]) bool { return !n.predicate.Update(e) } -func (n not) Delete(e event.DeleteEvent) bool { +func (n not[object]) Delete(e event.TypedDeleteEvent[object]) bool { return !n.predicate.Delete(e) } -func (n not) Generic(e event.GenericEvent) bool { +func (n not[object]) Generic(e event.TypedGenericEvent[object]) bool { return !n.predicate.Generic(e) } @@ -358,3 +415,15 @@ func LabelSelectorPredicate(s metav1.LabelSelector) (Predicate, error) { return selector.Matches(labels.Set(o.GetLabels())) }), nil } + +func isNil(arg any) bool { + if v := reflect.ValueOf(arg); !v.IsValid() || ((v.Kind() == reflect.Ptr || + v.Kind() == reflect.Interface || + v.Kind() == reflect.Slice || + v.Kind() == reflect.Map || + v.Kind() == reflect.Chan || + v.Kind() == reflect.Func) && v.IsNil()) { + return true + } + return false +} diff --git a/pkg/reconcile/reconcile.go b/pkg/reconcile/reconcile.go index d51cfc34ab..88303ae781 100644 --- a/pkg/reconcile/reconcile.go +++ b/pkg/reconcile/reconcile.go @@ -19,19 +19,36 @@ package reconcile import ( "context" "errors" + "reflect" "time" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) // Result contains the result of a Reconciler invocation. type Result struct { - // Requeue tells the Controller to requeue the reconcile key. Defaults to false. + // Requeue tells the Controller to perform a ratelimited requeue + // using the workqueues ratelimiter. Defaults to false. + // + // This setting is deprecated as it causes confusion and there is + // no good reason to use it. When waiting for an external event to + // happen, either the duration until it is supposed to happen or an + // appropriate poll interval should be used, rather than an + // interval emitted by a ratelimiter whose purpose it is to control + // retry on error. + // + // Deprecated: Use `RequeueAfter` instead. Requeue bool // RequeueAfter if greater than 0, tells the Controller to requeue the reconcile key after the Duration. // Implies that Requeue is true, there is no need to set Requeue to true at the same time as RequeueAfter. RequeueAfter time.Duration + + // Priority is the priority that will be used if the item gets re-enqueued (also if an error is returned). + // If Priority is not set the original Priority of the request is preserved. + // Note: Priority is only respected if the controller is using a priorityqueue.PriorityQueue. + Priority *int } // IsZero returns true if this result is empty. @@ -87,20 +104,70 @@ driven by actual cluster state read from the apiserver or a local cache. For example if responding to a Pod Delete Event, the Request won't contain that a Pod was deleted, instead the reconcile function observes this when reading the cluster state and seeing the Pod as missing. */ -type Reconciler interface { +type Reconciler = TypedReconciler[Request] + +// TypedReconciler implements an API for a specific Resource by Creating, Updating or Deleting Kubernetes +// objects, or by making changes to systems external to the cluster (e.g. cloudproviders, github, etc). +// +// The request type is what event handlers put into the workqueue. The workqueue then de-duplicates identical +// requests. +type TypedReconciler[request comparable] interface { // Reconcile performs a full reconciliation for the object referred to by the Request. - // The Controller will requeue the Request to be processed again if an error is non-nil or - // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. - Reconcile(context.Context, Request) (Result, error) + // + // If the returned error is non-nil, the Result is ignored and the request will be + // requeued using exponential backoff. The only exception is if the error is a + // TerminalError in which case no requeuing happens. + // + // If the error is nil and the returned Result has a non-zero result.RequeueAfter, the request + // will be requeued after the specified duration. + // + // If the error is nil and result.RequeueAfter is zero and result.Requeue is true, the request + // will be requeued using exponential backoff. + Reconcile(context.Context, request) (Result, error) } // Func is a function that implements the reconcile interface. -type Func func(context.Context, Request) (Result, error) +type Func = TypedFunc[Request] + +// TypedFunc is a function that implements the reconcile interface. +type TypedFunc[request comparable] func(context.Context, request) (Result, error) var _ Reconciler = Func(nil) // Reconcile implements Reconciler. -func (r Func) Reconcile(ctx context.Context, o Request) (Result, error) { return r(ctx, o) } +func (r TypedFunc[request]) Reconcile(ctx context.Context, req request) (Result, error) { + return r(ctx, req) +} + +// ObjectReconciler is a specialized version of Reconciler that acts on instances of client.Object. Each reconciliation +// event gets the associated object from Kubernetes before passing it to Reconcile. An ObjectReconciler can be used in +// Builder.Complete by calling AsReconciler. See Reconciler for more details. +type ObjectReconciler[object client.Object] interface { + Reconcile(context.Context, object) (Result, error) +} + +// AsReconciler creates a Reconciler based on the given ObjectReconciler. +func AsReconciler[object client.Object](client client.Client, rec ObjectReconciler[object]) Reconciler { + return &objectReconcilerAdapter[object]{ + objReconciler: rec, + client: client, + } +} + +type objectReconcilerAdapter[object client.Object] struct { + objReconciler ObjectReconciler[object] + client client.Client +} + +// Reconcile implements Reconciler. +func (a *objectReconcilerAdapter[object]) Reconcile(ctx context.Context, req Request) (Result, error) { + o := reflect.New(reflect.TypeOf(*new(object)).Elem()).Interface().(object) + if err := a.client.Get(ctx, req.NamespacedName, o); err != nil { + return Result{}, client.IgnoreNotFound(err) + } + + return a.objReconciler.Reconcile(ctx, o) +} // TerminalError is an error that will not be retried but still be logged // and recorded in metrics. @@ -112,11 +179,15 @@ type terminalError struct { err error } +// Unwrap returns nil if te.err is nil. func (te *terminalError) Unwrap() error { return te.err } func (te *terminalError) Error() string { + if te.err == nil { + return "nil terminal error" + } return "terminal error: " + te.err.Error() } diff --git a/pkg/reconcile/reconcile_test.go b/pkg/reconcile/reconcile_test.go index b5660f1b4f..bb5644b87c 100644 --- a/pkg/reconcile/reconcile_test.go +++ b/pkg/reconcile/reconcile_test.go @@ -23,11 +23,23 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) +type mockObjectReconciler struct { + reconcileFunc func(context.Context, *corev1.ConfigMap) (reconcile.Result, error) +} + +func (r *mockObjectReconciler) Reconcile(ctx context.Context, cm *corev1.ConfigMap) (reconcile.Result, error) { + return r.reconcileFunc(ctx, cm) +} + var _ = Describe("reconcile", func() { Describe("Result", func() { It("IsZero should return true if empty", func() { @@ -51,7 +63,7 @@ var _ = Describe("reconcile", func() { }) Describe("Func", func() { - It("should call the function with the request and return a nil error.", func() { + It("should call the function with the request and return a nil error.", func(ctx SpecContext) { request := reconcile.Request{ NamespacedName: types.NamespacedName{Name: "foo", Namespace: "bar"}, } @@ -65,12 +77,12 @@ var _ = Describe("reconcile", func() { return result, nil }) - actualResult, actualErr := instance.Reconcile(context.Background(), request) + actualResult, actualErr := instance.Reconcile(ctx, request) Expect(actualResult).To(Equal(result)) Expect(actualErr).NotTo(HaveOccurred()) }) - It("should call the function with the request and return an error.", func() { + It("should call the function with the request and return an error.", func(ctx SpecContext) { request := reconcile.Request{ NamespacedName: types.NamespacedName{Name: "foo", Namespace: "bar"}, } @@ -85,7 +97,7 @@ var _ = Describe("reconcile", func() { return result, err }) - actualResult, actualErr := instance.Reconcile(context.Background(), request) + actualResult, actualErr := instance.Reconcile(ctx, request) Expect(actualResult).To(Equal(result)) Expect(actualErr).To(Equal(err)) }) @@ -96,5 +108,81 @@ var _ = Describe("reconcile", func() { Expect(apierrors.IsGone(terminalError)).To(BeTrue()) }) + + It("should handle nil terminal errors properly", func() { + err := reconcile.TerminalError(nil) + Expect(err.Error()).To(Equal("nil terminal error")) + }) + }) + + Describe("AsReconciler", func() { + var testenv *envtest.Environment + var testClient client.Client + + BeforeEach(func() { + testenv = &envtest.Environment{} + + cfg, err := testenv.Start() + Expect(err).NotTo(HaveOccurred()) + + testClient, err = client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + Expect(testenv.Stop()).NotTo(HaveOccurred()) + }) + + Context("with an existing object", func() { + var key client.ObjectKey + + BeforeEach(func(ctx SpecContext) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + key = client.ObjectKeyFromObject(cm) + + err := testClient.Create(ctx, cm) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should Get the object and call the ObjectReconciler", func(ctx SpecContext) { + var actual *corev1.ConfigMap + reconciler := reconcile.AsReconciler(testClient, &mockObjectReconciler{ + reconcileFunc: func(ctx context.Context, cm *corev1.ConfigMap) (reconcile.Result, error) { + actual = cm + return reconcile.Result{}, nil + }, + }) + + res, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(BeZero()) + Expect(actual).NotTo(BeNil()) + Expect(actual.ObjectMeta.Name).To(Equal(key.Name)) + Expect(actual.ObjectMeta.Namespace).To(Equal(key.Namespace)) + }) + }) + + Context("with an object that doesn't exist", func() { + It("should not call the ObjectReconciler", func(ctx SpecContext) { + called := false + reconciler := reconcile.AsReconciler(testClient, &mockObjectReconciler{ + reconcileFunc: func(ctx context.Context, cm *corev1.ConfigMap) (reconcile.Result, error) { + called = true + return reconcile.Result{}, nil + }, + }) + + key := types.NamespacedName{Namespace: "default", Name: "fake-obj"} + res, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: key}) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(BeZero()) + Expect(called).To(BeFalse()) + }) + }) }) }) diff --git a/pkg/source/example_test.go b/pkg/source/example_test.go index 77857729de..b596ff0a0a 100644 --- a/pkg/source/example_test.go +++ b/pkg/source/example_test.go @@ -31,7 +31,7 @@ var ctrl controller.Controller // This example Watches for Pod Events (e.g. Create / Update / Delete) and enqueues a reconcile.Request // with the Name and Namespace of the Pod. func ExampleKind() { - err := ctrl.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.EnqueueRequestForObject{}) + err := ctrl.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}, &handler.TypedEnqueueRequestForObject[*corev1.Pod]{})) if err != nil { // handle it } @@ -43,8 +43,10 @@ func ExampleChannel() { events := make(chan event.GenericEvent) err := ctrl.Watch( - &source.Channel{Source: events}, - &handler.EnqueueRequestForObject{}, + source.Channel( + events, + &handler.EnqueueRequestForObject{}, + ), ) if err != nil { // handle it diff --git a/pkg/source/source.go b/pkg/source/source.go index 099c8d68fa..c2c2dc4e07 100644 --- a/pkg/source/source.go +++ b/pkg/source/source.go @@ -18,94 +18,178 @@ package source import ( "context" + "errors" "fmt" "sync" + toolscache "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" internal "sigs.k8s.io/controller-runtime/pkg/internal/source" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/predicate" ) -const ( - // defaultBufferSize is the default number of event notifications that can be buffered. - defaultBufferSize = 1024 -) +var logInformer = logf.RuntimeLog.WithName("source").WithName("Informer") -// Source is a source of events (eh.g. Create, Update, Delete operations on Kubernetes Objects, Webhook callbacks, etc) +// Source is a source of events (e.g. Create, Update, Delete operations on Kubernetes Objects, Webhook callbacks, etc) // which should be processed by event.EventHandlers to enqueue reconcile.Requests. // // * Use Kind for events originating in the cluster (e.g. Pod Create, Pod Update, Deployment Update). // -// * Use Channel for events originating outside the cluster (eh.g. GitHub Webhook callback, Polling external urls). +// * Use Channel for events originating outside the cluster (e.g. GitHub Webhook callback, Polling external urls). // // Users may build their own Source implementations. -type Source interface { - // Start is internal and should be called only by the Controller to register an EventHandler with the Informer - // to enqueue reconcile.Requests. - Start(context.Context, handler.EventHandler, workqueue.RateLimitingInterface, ...predicate.Predicate) error +type Source = TypedSource[reconcile.Request] + +// TypedSource is a generic source of events (e.g. Create, Update, Delete operations on Kubernetes Objects, Webhook callbacks, etc) +// which should be processed by event.EventHandlers to enqueue a request. +// +// * Use Kind for events originating in the cluster (e.g. Pod Create, Pod Update, Deployment Update). +// +// * Use Channel for events originating outside the cluster (e.g. GitHub Webhook callback, Polling external urls). +// +// Users may build their own Source implementations. +type TypedSource[request comparable] interface { + // Start is internal and should be called only by the Controller to start the source. + // Start must be non-blocking. + Start(context.Context, workqueue.TypedRateLimitingInterface[request]) error } // SyncingSource is a source that needs syncing prior to being usable. The controller // will call its WaitForSync prior to starting workers. -type SyncingSource interface { - Source +type SyncingSource = TypedSyncingSource[reconcile.Request] + +// TypedSyncingSource is a source that needs syncing prior to being usable. The controller +// will call its WaitForSync prior to starting workers. +type TypedSyncingSource[request comparable] interface { + TypedSource[request] WaitForSync(ctx context.Context) error } // Kind creates a KindSource with the given cache provider. -func Kind(cache cache.Cache, object client.Object) SyncingSource { - return &internal.Kind{Type: object, Cache: cache} +func Kind[object client.Object]( + cache cache.Cache, + obj object, + handler handler.TypedEventHandler[object, reconcile.Request], + predicates ...predicate.TypedPredicate[object], +) SyncingSource { + return TypedKind(cache, obj, handler, predicates...) } -var _ Source = &Channel{} +// TypedKind creates a KindSource with the given cache provider. +func TypedKind[object client.Object, request comparable]( + cache cache.Cache, + obj object, + handler handler.TypedEventHandler[object, request], + predicates ...predicate.TypedPredicate[object], +) TypedSyncingSource[request] { + return &internal.Kind[object, request]{ + Type: obj, + Cache: cache, + Handler: handler, + Predicates: predicates, + } +} + +var _ Source = &channel[string, reconcile.Request]{} + +// ChannelOpt allows to configure a source.Channel. +type ChannelOpt[object any, request comparable] func(*channel[object, request]) + +// WithPredicates adds the configured predicates to a source.Channel. +func WithPredicates[object any, request comparable](p ...predicate.TypedPredicate[object]) ChannelOpt[object, request] { + return func(c *channel[object, request]) { + c.predicates = append(c.predicates, p...) + } +} + +// WithBufferSize configures the buffer size for a source.Channel. By +// default, the buffer size is 1024. +func WithBufferSize[object any, request comparable](bufferSize int) ChannelOpt[object, request] { + return func(c *channel[object, request]) { + c.bufferSize = &bufferSize + } +} // Channel is used to provide a source of events originating outside the cluster // (e.g. GitHub Webhook callback). Channel requires the user to wire the external -// source (eh.g. http handler) to write GenericEvents to the underlying channel. -type Channel struct { +// source (e.g. http handler) to write GenericEvents to the underlying channel. +func Channel[object any]( + source <-chan event.TypedGenericEvent[object], + handler handler.TypedEventHandler[object, reconcile.Request], + opts ...ChannelOpt[object, reconcile.Request], +) Source { + return TypedChannel[object, reconcile.Request](source, handler, opts...) +} + +// TypedChannel is used to provide a source of events originating outside the cluster +// (e.g. GitHub Webhook callback). Channel requires the user to wire the external +// source (e.g. http handler) to write GenericEvents to the underlying channel. +func TypedChannel[object any, request comparable]( + source <-chan event.TypedGenericEvent[object], + handler handler.TypedEventHandler[object, request], + opts ...ChannelOpt[object, request], +) TypedSource[request] { + c := &channel[object, request]{ + source: source, + handler: handler, + } + for _, opt := range opts { + opt(c) + } + + return c +} + +type channel[object any, request comparable] struct { // once ensures the event distribution goroutine will be performed only once once sync.Once - // Source is the source channel to fetch GenericEvents - Source <-chan event.GenericEvent + // source is the source channel to fetch GenericEvents + source <-chan event.TypedGenericEvent[object] - // dest is the destination channels of the added event handlers - dest []chan event.GenericEvent + handler handler.TypedEventHandler[object, request] + + predicates []predicate.TypedPredicate[object] - // DestBufferSize is the specified buffer size of dest channels. - // Default to 1024 if not specified. - DestBufferSize int + bufferSize *int + + // dest is the destination channels of the added event handlers + dest []chan event.TypedGenericEvent[object] // destLock is to ensure the destination channels are safely added/removed destLock sync.Mutex } -func (cs *Channel) String() string { +func (cs *channel[object, request]) String() string { return fmt.Sprintf("channel source: %p", cs) } // Start implements Source and should only be called by the Controller. -func (cs *Channel) Start( +func (cs *channel[object, request]) Start( ctx context.Context, - handler handler.EventHandler, - queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { + queue workqueue.TypedRateLimitingInterface[request], +) error { // Source should have been specified by the user. - if cs.Source == nil { + if cs.source == nil { return fmt.Errorf("must specify Channel.Source") } + if cs.handler == nil { + return errors.New("must specify Channel.Handler") + } - // use default value if DestBufferSize not specified - if cs.DestBufferSize == 0 { - cs.DestBufferSize = defaultBufferSize + if cs.bufferSize == nil { + cs.bufferSize = ptr.To(1024) } - dst := make(chan event.GenericEvent, cs.DestBufferSize) + dst := make(chan event.TypedGenericEvent[object], *cs.bufferSize) cs.destLock.Lock() cs.dest = append(cs.dest, dst) @@ -119,7 +203,7 @@ func (cs *Channel) Start( go func() { for evt := range dst { shouldHandle := true - for _, p := range prct { + for _, p := range cs.predicates { if !p.Generic(evt) { shouldHandle = false break @@ -130,7 +214,7 @@ func (cs *Channel) Start( func() { ctx, cancel := context.WithCancel(ctx) defer cancel() - handler.Generic(ctx, evt, queue) + cs.handler.Generic(ctx, evt, queue) }() } } @@ -139,7 +223,7 @@ func (cs *Channel) Start( return nil } -func (cs *Channel) doStop() { +func (cs *channel[object, request]) doStop() { cs.destLock.Lock() defer cs.destLock.Unlock() @@ -148,7 +232,7 @@ func (cs *Channel) doStop() { } } -func (cs *Channel) distribute(evt event.GenericEvent) { +func (cs *channel[object, request]) distribute(evt event.TypedGenericEvent[object]) { cs.destLock.Lock() defer cs.destLock.Unlock() @@ -162,14 +246,14 @@ func (cs *Channel) distribute(evt event.GenericEvent) { } } -func (cs *Channel) syncLoop(ctx context.Context) { +func (cs *channel[object, request]) syncLoop(ctx context.Context) { for { select { case <-ctx.Done(): // Close destination channels cs.doStop() return - case evt, stillOpen := <-cs.Source: + case evt, stillOpen := <-cs.source: if !stillOpen { // if the source channel is closed, we're never gonna get // anything more on it, so stop & bail @@ -184,21 +268,27 @@ func (cs *Channel) syncLoop(ctx context.Context) { // Informer is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create). type Informer struct { // Informer is the controller-runtime Informer - Informer cache.Informer + Informer cache.Informer + Handler handler.EventHandler + Predicates []predicate.Predicate } var _ Source = &Informer{} // Start is internal and should be called only by the Controller to register an EventHandler with the Informer // to enqueue reconcile.Requests. -func (is *Informer) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { +func (is *Informer) Start(ctx context.Context, queue workqueue.TypedRateLimitingInterface[reconcile.Request]) error { // Informer should have been specified by the user. if is.Informer == nil { return fmt.Errorf("must specify Informer.Informer") } + if is.Handler == nil { + return errors.New("must specify Informer.Handler") + } - _, err := is.Informer.AddEventHandler(internal.NewEventHandler(ctx, queue, handler, prct).HandlerFuncs()) + _, err := is.Informer.AddEventHandlerWithOptions(internal.NewEventHandler(ctx, queue, is.Handler, is.Predicates), toolscache.HandlerOptions{ + Logger: &logInformer, + }) if err != nil { return err } @@ -212,14 +302,16 @@ func (is *Informer) String() string { var _ Source = Func(nil) // Func is a function that implements Source. -type Func func(context.Context, handler.EventHandler, workqueue.RateLimitingInterface, ...predicate.Predicate) error +type Func = TypedFunc[reconcile.Request] + +// TypedFunc is a function that implements Source. +type TypedFunc[request comparable] func(context.Context, workqueue.TypedRateLimitingInterface[request]) error // Start implements Source. -func (f Func) Start(ctx context.Context, evt handler.EventHandler, queue workqueue.RateLimitingInterface, - pr ...predicate.Predicate) error { - return f(ctx, evt, queue, pr...) +func (f TypedFunc[request]) Start(ctx context.Context, queue workqueue.TypedRateLimitingInterface[request]) error { + return f(ctx, queue) } -func (f Func) String() string { +func (f TypedFunc[request]) String() string { return fmt.Sprintf("func source: %p", f) } diff --git a/pkg/source/source_integration_test.go b/pkg/source/source_integration_test.go index 594d3c9a9c..cc0ba530ec 100644 --- a/pkg/source/source_integration_test.go +++ b/pkg/source/source_integration_test.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" . "github.com/onsi/ginkgo/v2" @@ -39,12 +40,12 @@ import ( var _ = Describe("Source", func() { var instance1, instance2 source.Source var obj client.Object - var q workqueue.RateLimitingInterface + var q workqueue.TypedRateLimitingInterface[reconcile.Request] var c1, c2 chan interface{} var ns string count := 0 - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { // Create the namespace for the test ns = fmt.Sprintf("controller-source-kindsource-%v", count) count++ @@ -53,17 +54,16 @@ var _ = Describe("Source", func() { }, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - q = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q = workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) c1 = make(chan interface{}) c2 = make(chan interface{}) }) - JustBeforeEach(func() { - instance1 = source.Kind(icache, obj) - instance2 = source.Kind(icache, obj) - }) - - AfterEach(func() { + AfterEach(func(ctx SpecContext) { err := clientset.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) close(c1) @@ -74,7 +74,7 @@ var _ = Describe("Source", func() { Context("for a Deployment resource", func() { obj = &appsv1.Deployment{} - It("should provide Deployment Events", func() { + It("should provide Deployment Events", func(ctx SpecContext) { var created, updated, deleted *appsv1.Deployment var err error @@ -103,17 +103,17 @@ var _ = Describe("Source", func() { // Create an event handler to verify the events newHandler := func(c chan interface{}) handler.Funcs { return handler.Funcs{ - CreateFunc: func(ctx context.Context, evt event.CreateEvent, rli workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, evt event.CreateEvent, rli workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt }, - UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, rli workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, rli workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt }, - DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, rli workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, rli workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt @@ -124,8 +124,10 @@ var _ = Describe("Source", func() { handler2 := newHandler(c2) // Create 2 instances - Expect(instance1.Start(ctx, handler1, q)).To(Succeed()) - Expect(instance2.Start(ctx, handler2, q)).To(Succeed()) + instance1 = source.Kind(icache, obj, handler1) + instance2 = source.Kind(icache, obj, handler2) + Expect(instance1.Start(ctx, q)).To(Succeed()) + Expect(instance2.Start(ctx, q)).To(Succeed()) By("Creating a Deployment and expecting the CreateEvent.") created, err = client.Create(ctx, deployment, metav1.CreateOptions{}) @@ -237,35 +239,42 @@ var _ = Describe("Source", func() { }) Context("for a ReplicaSet resource", func() { - It("should provide a ReplicaSet CreateEvent", func() { + It("should provide a ReplicaSet CreateEvent", func(ctx SpecContext) { c := make(chan struct{}) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Informer{Informer: depInformer} - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - var err error - rs, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(rs)) - close(c) - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected GenericEvent") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := &source.Informer{ + Informer: depInformer, + Handler: handler.Funcs{ + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + var err error + rs, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + Expect(q2).To(BeIdenticalTo(q)) + Expect(evt.Object).To(Equal(rs)) + close(c) + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected GenericEvent") + }, }, - }, q) + } + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) _, err = clientset.AppsV1().ReplicaSets("default").Create(ctx, rs, metav1.CreateOptions{}) @@ -273,7 +282,7 @@ var _ = Describe("Source", func() { <-c }) - It("should provide a ReplicaSet UpdateEvent", func() { + It("should provide a ReplicaSet UpdateEvent", func(ctx SpecContext) { var err error rs, err = clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -281,33 +290,40 @@ var _ = Describe("Source", func() { rs2 := rs.DeepCopy() rs2.SetLabels(map[string]string{"biz": "baz"}) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Informer{Informer: depInformer} - err = instance.Start(ctx, handler.Funcs{ - CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { - }, - UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - var err error - rs2, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := &source.Informer{ + Informer: depInformer, + Handler: handler.Funcs{ + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + }, + UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + var err error + rs2, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) - Expect(q2).To(Equal(q)) - Expect(evt.ObjectOld).To(Equal(rs)) + Expect(q2).To(Equal(q)) + Expect(evt.ObjectOld).To(Equal(rs)) - Expect(evt.ObjectNew).To(Equal(rs2)) + Expect(evt.ObjectNew).To(Equal(rs2)) - close(c) - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected GenericEvent") + close(c) + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected GenericEvent") + }, }, - }, q) + } + err = instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) _, err = clientset.AppsV1().ReplicaSets("default").Update(ctx, rs2, metav1.UpdateOptions{}) @@ -315,27 +331,34 @@ var _ = Describe("Source", func() { <-c }) - It("should provide a ReplicaSet DeletedEvent", func() { + It("should provide a ReplicaSet DeletedEvent", func(ctx SpecContext) { c := make(chan struct{}) - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Informer{Informer: depInformer} - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - }, - DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Expect(q2).To(Equal(q)) - Expect(evt.Object.GetName()).To(Equal(rs.Name)) - close(c) - }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected GenericEvent") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := &source.Informer{ + Informer: depInformer, + Handler: handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + }, + DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Expect(q2).To(Equal(q)) + Expect(evt.Object.GetName()).To(Equal(rs.Name)) + close(c) + }, + GenericFunc: func(context.Context, event.GenericEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected GenericEvent") + }, }, - }, q) + } + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) err = clientset.AppsV1().ReplicaSets("default").Delete(ctx, rs.Name, metav1.DeleteOptions{}) diff --git a/pkg/source/source_suite_test.go b/pkg/source/source_suite_test.go index 131099f0b9..774d978ca3 100644 --- a/pkg/source/source_suite_test.go +++ b/pkg/source/source_suite_test.go @@ -39,11 +39,12 @@ var testenv *envtest.Environment var config *rest.Config var clientset *kubernetes.Clientset var icache cache.Cache -var ctx context.Context var cancel context.CancelFunc var _ = BeforeSuite(func() { - ctx, cancel = context.WithCancel(context.Background()) + var ctx context.Context + // Has to be derived from context.Background, as it stays valid past the BeforeSuite + ctx, cancel = context.WithCancel(context.Background()) //nolint:forbidigo logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} diff --git a/pkg/source/source_test.go b/pkg/source/source_test.go index f5e231d540..ad311d73b2 100644 --- a/pkg/source/source_test.go +++ b/pkg/source/source_test.go @@ -23,10 +23,13 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/cache/informertest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" corev1 "k8s.io/api/core/v1" @@ -53,7 +56,7 @@ var _ = Describe("Source", func() { }) Context("for a Pod resource", func() { - It("should provide a Pod CreateEvent", func() { + It("should provide a Pod CreateEvent", func(ctx SpecContext) { c := make(chan struct{}) p := &corev1.Pod{ Spec: corev1.PodSpec{ @@ -63,51 +66,58 @@ var _ = Describe("Source", func() { }, } - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := source.Kind(ic, &corev1.Pod{}) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(ctx context.Context, evt event.TypedCreateEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt.Object).To(Equal(p)) close(c) }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.TypedUpdateEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.TypedDeleteEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.TypedGenericEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, - }, q) + }) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).NotTo(HaveOccurred()) + Expect(instance.WaitForSync(ctx)).NotTo(HaveOccurred()) - i, err := ic.FakeInformerFor(&corev1.Pod{}) + i, err := ic.FakeInformerFor(ctx, &corev1.Pod{}) Expect(err).NotTo(HaveOccurred()) i.Add(p) <-c }) - It("should provide a Pod UpdateEvent", func() { + It("should provide a Pod UpdateEvent", func(ctx SpecContext) { p2 := p.DeepCopy() p2.SetLabels(map[string]string{"biz": "baz"}) ic := &informertest.FakeInformers{} - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := source.Kind(ic, &corev1.Pod{}) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(ctx context.Context, evt event.TypedCreateEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, evt event.TypedUpdateEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.ObjectOld).To(Equal(p)) @@ -116,26 +126,27 @@ var _ = Describe("Source", func() { close(c) }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.TypedDeleteEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.TypedGenericEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, - }, q) + }) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).NotTo(HaveOccurred()) + Expect(instance.WaitForSync(ctx)).NotTo(HaveOccurred()) - i, err := ic.FakeInformerFor(&corev1.Pod{}) + i, err := ic.FakeInformerFor(ctx, &corev1.Pod{}) Expect(err).NotTo(HaveOccurred()) i.Update(p, p2) <-c }) - It("should provide a Pod DeletedEvent", func() { + It("should provide a Pod DeletedEvent", func(ctx SpecContext) { c := make(chan struct{}) p := &corev1.Pod{ Spec: corev1.PodSpec{ @@ -145,32 +156,36 @@ var _ = Describe("Source", func() { }, } - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := source.Kind(ic, &corev1.Pod{}) - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{ + CreateFunc: func(context.Context, event.TypedCreateEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.TypedUpdateEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, evt event.TypedDeleteEvent[*corev1.Pod], q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.Object).To(Equal(p)) close(c) }, - GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.TypedGenericEvent[*corev1.Pod], workqueue.TypedRateLimitingInterface[reconcile.Request]) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, - }, q) + }) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).NotTo(HaveOccurred()) + Expect(instance.WaitForSync(ctx)).NotTo(HaveOccurred()) - i, err := ic.FakeInformerFor(&corev1.Pod{}) + i, err := ic.FakeInformerFor(ctx, &corev1.Pod{}) Expect(err).NotTo(HaveOccurred()) i.Delete(p) @@ -178,50 +193,60 @@ var _ = Describe("Source", func() { }) }) - It("should return an error from Start cache was not provided", func() { - instance := source.Kind(nil, &corev1.Pod{}) - err := instance.Start(ctx, nil, nil) + It("should return an error from Start cache was not provided", func(ctx SpecContext) { + instance := source.Kind(nil, &corev1.Pod{}, nil) + err := instance.Start(ctx, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("must create Kind with a non-nil cache")) }) - It("should return an error from Start if a type was not provided", func() { - instance := source.Kind(ic, nil) - err := instance.Start(ctx, nil, nil) + It("should return an error from Start if a type was not provided", func(ctx SpecContext) { + instance := source.Kind[client.Object](ic, nil, nil) + err := instance.Start(ctx, nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("must create Kind with a non-nil object")) }) + It("should return an error from Start if a handler was not provided", func(ctx SpecContext) { + instance := source.Kind(ic, &corev1.Pod{}, nil) + err := instance.Start(ctx, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must create Kind with non-nil handler")) + }) - It("should return an error if syncing fails", func() { + It("should return an error if syncing fails", func(ctx SpecContext) { f := false - instance := source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}) - Expect(instance.Start(context.Background(), nil, nil)).NotTo(HaveOccurred()) - err := instance.WaitForSync(context.Background()) + instance := source.Kind[client.Object](&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}, &handler.EnqueueRequestForObject{}) + Expect(instance.Start(ctx, nil)).NotTo(HaveOccurred()) + err := instance.WaitForSync(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("cache did not sync")) }) Context("for a Kind not in the cache", func() { - It("should return an error when WaitForSync is called", func() { + It("should return an error when WaitForSync is called", func(specContext SpecContext) { ic.Error = fmt.Errorf("test error") - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) - ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + ctx, cancel := context.WithTimeout(specContext, 2*time.Second) defer cancel() - instance := source.Kind(ic, &corev1.Pod{}) - err := instance.Start(ctx, handler.Funcs{}, q) + instance := source.Kind(ic, &corev1.Pod{}, handler.TypedFuncs[*corev1.Pod, reconcile.Request]{}) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) - Eventually(instance.WaitForSync).WithArguments(context.Background()).Should(HaveOccurred()) + Eventually(instance.WaitForSync).WithArguments(ctx).Should(HaveOccurred()) }) }) - It("should return an error if syncing fails", func() { + It("should return an error if syncing fails", func(ctx SpecContext) { f := false - instance := source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}) - Expect(instance.Start(context.Background(), nil, nil)).NotTo(HaveOccurred()) - err := instance.WaitForSync(context.Background()) + instance := source.Kind[client.Object](&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}, &handler.EnqueueRequestForObject{}) + Expect(instance.Start(ctx, nil)).NotTo(HaveOccurred()) + err := instance.WaitForSync(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("cache did not sync")) @@ -229,46 +254,40 @@ var _ = Describe("Source", func() { }) Describe("Func", func() { - It("should be called from Start", func() { + It("should be called from Start", func(ctx SpecContext) { run := false instance := source.Func(func( context.Context, - handler.EventHandler, - workqueue.RateLimitingInterface, ...predicate.Predicate) error { + workqueue.TypedRateLimitingInterface[reconcile.Request]) error { run = true return nil }) - Expect(instance.Start(ctx, nil, nil)).NotTo(HaveOccurred()) + Expect(instance.Start(ctx, nil)).NotTo(HaveOccurred()) Expect(run).To(BeTrue()) expected := fmt.Errorf("expected error: Func") instance = source.Func(func( context.Context, - handler.EventHandler, - workqueue.RateLimitingInterface, ...predicate.Predicate) error { + workqueue.TypedRateLimitingInterface[reconcile.Request]) error { return expected }) - Expect(instance.Start(ctx, nil, nil)).To(Equal(expected)) + Expect(instance.Start(ctx, nil)).To(Equal(expected)) }) }) Describe("Channel", func() { - var ctx context.Context - var cancel context.CancelFunc var ch chan event.GenericEvent BeforeEach(func() { - ctx, cancel = context.WithCancel(context.Background()) ch = make(chan event.GenericEvent) }) AfterEach(func() { - cancel() close(ch) }) Context("for a source", func() { - It("should provide a GenericEvent", func() { + It("should provide a GenericEvent", func(ctx SpecContext) { ch := make(chan event.GenericEvent) c := make(chan struct{}) p := &corev1.Pod{ @@ -287,73 +306,87 @@ var _ = Describe("Source", func() { }, } - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{Source: ch} - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - // The empty event should have been filtered out by the predicates, - // and will not be passed to the handler. - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(p)) - close(c) + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + // The empty event should have been filtered out by the predicates, + // and will not be passed to the handler. + Expect(q2).To(BeIdenticalTo(q)) + Expect(evt.Object).To(Equal(p)) + close(c) + }, }, - }, q, prct) + source.WithPredicates[client.Object, reconcile.Request](prct), + ) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) ch <- invalidEvt ch <- evt <-c }) - It("should get pending events processed once channel unblocked", func() { + It("should get pending events processed once channel unblocked", func(ctx SpecContext) { ch := make(chan event.GenericEvent) unblock := make(chan struct{}) processed := make(chan struct{}) evt := event.GenericEvent{} eventCount := 0 - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) // Add a handler to get distribution blocked - instance := &source.Channel{Source: ch} - instance.DestBufferSize = 1 - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - // Block for the first time - if eventCount == 0 { - <-unblock - } - eventCount++ - - if eventCount == 3 { - close(processed) - } + instance := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + // Block for the first time + if eventCount == 0 { + <-unblock + } + eventCount++ + + if eventCount == 3 { + close(processed) + } + }, }, - }, q) + ) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) // Write 3 events into the source channel. @@ -374,42 +407,52 @@ var _ = Describe("Source", func() { // Validate all of the events have been processed. Expect(eventCount).To(Equal(3)) }) - It("should be able to cope with events in the channel before the source is started", func() { + It("should be able to cope with events in the channel before the source is started", func(ctx SpecContext) { ch := make(chan event.GenericEvent, 1) processed := make(chan struct{}) evt := event.GenericEvent{} ch <- evt - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) // Add a handler to get distribution blocked - instance := &source.Channel{Source: ch} - instance.DestBufferSize = 1 + instance := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") + close(processed) + }, }, - GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() + ) - close(processed) - }, - }, q) + err := instance.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) <-processed }) - It("should stop when the source channel is closed", func() { - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") + It("should stop when the source channel is closed", func(ctx SpecContext) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) // if we didn't stop, we'd start spamming the queue with empty // messages as we "received" a zero-valued GenericEvent from // the source channel @@ -421,112 +464,48 @@ var _ = Describe("Source", func() { close(ch) By("feeding that channel to a channel source") - src := &source.Channel{Source: ch} - processed := make(chan struct{}) defer close(processed) + src := source.Channel( + ch, + handler.Funcs{ + CreateFunc: func(context.Context, event.CreateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected CreateEvent") + }, + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected UpdateEvent") + }, + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() + Fail("Unexpected DeleteEvent") + }, + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.TypedRateLimitingInterface[reconcile.Request]) { + defer GinkgoRecover() - err := src.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") + processed <- struct{}{} + }, }, - GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() + ) - processed <- struct{}{} - }, - }, q) + err := src.Start(ctx, q) Expect(err).NotTo(HaveOccurred()) By("expecting to only get one event") Eventually(processed).Should(Receive()) Consistently(processed).ShouldNot(Receive()) }) - It("should get error if no source specified", func() { - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{ /*no source specified*/ } - err := instance.Start(ctx, handler.Funcs{}, q) + It("should get error if no source specified", func(ctx SpecContext) { + q := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[reconcile.Request](), + workqueue.TypedRateLimitingQueueConfig[reconcile.Request]{ + Name: "test", + }) + instance := source.Channel[string](nil, nil /*no source specified*/) + err := instance.Start(ctx, q) Expect(err).To(Equal(fmt.Errorf("must specify Channel.Source"))) }) }) - Context("for multi sources (handlers)", func() { - It("should provide GenericEvents for all handlers", func() { - ch := make(chan event.GenericEvent) - p := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, - } - evt := event.GenericEvent{ - Object: p, - } - - var resEvent1, resEvent2 event.GenericEvent - c1 := make(chan struct{}) - c2 := make(chan struct{}) - - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{Source: ch} - err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(p)) - resEvent1 = evt - close(c1) - }, - }, q) - Expect(err).NotTo(HaveOccurred()) - - err = instance.Start(ctx, handler.Funcs{ - CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected CreateEvent") - }, - UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected UpdateEvent") - }, - DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Fail("Unexpected DeleteEvent") - }, - GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { - defer GinkgoRecover() - Expect(q2).To(BeIdenticalTo(q)) - Expect(evt.Object).To(Equal(p)) - resEvent2 = evt - close(c2) - }, - }, q) - Expect(err).NotTo(HaveOccurred()) - - ch <- evt - <-c1 - <-c2 - - // Validate the two handlers received same event - Expect(resEvent1).To(Equal(resEvent2)) - }) - }) }) }) diff --git a/pkg/webhook/admission/decode.go b/pkg/webhook/admission/decode.go index f14f130f7b..55f1cafb5e 100644 --- a/pkg/webhook/admission/decode.go +++ b/pkg/webhook/admission/decode.go @@ -26,22 +26,35 @@ import ( // Decoder knows how to decode the contents of an admission // request into a concrete object. -type Decoder struct { +type Decoder interface { + // Decode decodes the inlined object in the AdmissionRequest into the passed-in runtime.Object. + // If you want decode the OldObject in the AdmissionRequest, use DecodeRaw. + // It errors out if req.Object.Raw is empty i.e. containing 0 raw bytes. + Decode(req Request, into runtime.Object) error + + // DecodeRaw decodes a RawExtension object into the passed-in runtime.Object. + // It errors out if rawObj is empty i.e. containing 0 raw bytes. + DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) error +} + +// decoder knows how to decode the contents of an admission +// request into a concrete object. +type decoder struct { codecs serializer.CodecFactory } -// NewDecoder creates a Decoder given the runtime.Scheme. -func NewDecoder(scheme *runtime.Scheme) *Decoder { +// NewDecoder creates a decoder given the runtime.Scheme. +func NewDecoder(scheme *runtime.Scheme) Decoder { if scheme == nil { panic("scheme should never be nil") } - return &Decoder{codecs: serializer.NewCodecFactory(scheme)} + return &decoder{codecs: serializer.NewCodecFactory(scheme)} } // Decode decodes the inlined object in the AdmissionRequest into the passed-in runtime.Object. // If you want decode the OldObject in the AdmissionRequest, use DecodeRaw. // It errors out if req.Object.Raw is empty i.e. containing 0 raw bytes. -func (d *Decoder) Decode(req Request, into runtime.Object) error { +func (d *decoder) Decode(req Request, into runtime.Object) error { // we error out if rawObj is an empty object. if len(req.Object.Raw) == 0 { return fmt.Errorf("there is no content to decode") @@ -51,7 +64,7 @@ func (d *Decoder) Decode(req Request, into runtime.Object) error { // DecodeRaw decodes a RawExtension object into the passed-in runtime.Object. // It errors out if rawObj is empty i.e. containing 0 raw bytes. -func (d *Decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) error { +func (d *decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) error { // NB(directxman12): there's a bug/weird interaction between decoders and // the API server where the API server doesn't send a GVK on the embedded // objects, which means the unstructured decoder refuses to decode. It @@ -71,6 +84,7 @@ func (d *Decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) er return err } unstructuredInto.SetUnstructuredContent(object) + return nil } deserializer := d.codecs.UniversalDeserializer() diff --git a/pkg/webhook/admission/decode_test.go b/pkg/webhook/admission/decode_test.go index 66586da790..130308800f 100644 --- a/pkg/webhook/admission/decode_test.go +++ b/pkg/webhook/admission/decode_test.go @@ -29,7 +29,7 @@ import ( ) var _ = Describe("Admission Webhook Decoder", func() { - var decoder *Decoder + var decoder Decoder BeforeEach(func() { By("creating a new decoder for a scheme") decoder = NewDecoder(scheme.Scheme) @@ -123,6 +123,8 @@ var _ = Describe("Admission Webhook Decoder", func() { })) }) + // NOTE: This will only pass if a GVK is provided. An unstructered object without a GVK may succeed + // in decoding to an alternate type. It("should fail to decode if the object in the request doesn't match the passed-in type", func() { By("trying to extract a pod from the quest into a node") Expect(decoder.Decode(req, &corev1.Node{})).NotTo(Succeed()) @@ -152,4 +154,35 @@ var _ = Describe("Admission Webhook Decoder", func() { "namespace": "default", })) }) + + req2 := Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: "CREATE", + Object: runtime.RawExtension{ + Raw: []byte(`{ + "metadata": { + "name": "foo", + "namespace": "default" + }, + "spec": { + "containers": [ + { + "image": "bar:v2", + "name": "bar" + } + ] + } + }`), + }, + OldObject: runtime.RawExtension{ + Object: nil, + }, + }, + } + + It("should decode a valid admission request without GVK", func() { + By("extracting the object from the request") + var target3 unstructured.Unstructured + Expect(decoder.DecodeRaw(req2.Object, &target3)).To(Succeed()) + }) }) diff --git a/pkg/webhook/admission/defaulter.go b/pkg/webhook/admission/defaulter.go deleted file mode 100644 index a3b7207168..0000000000 --- a/pkg/webhook/admission/defaulter.go +++ /dev/null @@ -1,82 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "context" - "encoding/json" - "net/http" - - admissionv1 "k8s.io/api/admission/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -// Defaulter defines functions for setting defaults on resources. -type Defaulter interface { - runtime.Object - Default() -} - -// DefaultingWebhookFor creates a new Webhook for Defaulting the provided type. -func DefaultingWebhookFor(scheme *runtime.Scheme, defaulter Defaulter) *Webhook { - return &Webhook{ - Handler: &mutatingHandler{defaulter: defaulter, decoder: NewDecoder(scheme)}, - } -} - -type mutatingHandler struct { - defaulter Defaulter - decoder *Decoder -} - -// Handle handles admission requests. -func (h *mutatingHandler) Handle(ctx context.Context, req Request) Response { - if h.decoder == nil { - panic("decoder should never be nil") - } - if h.defaulter == nil { - panic("defaulter should never be nil") - } - - // always skip when a DELETE operation received in mutation handler - // describe in https://github.com/kubernetes-sigs/controller-runtime/issues/1762 - if req.Operation == admissionv1.Delete { - return Response{AdmissionResponse: admissionv1.AdmissionResponse{ - Allowed: true, - Result: &metav1.Status{ - Code: http.StatusOK, - }, - }} - } - - // Get the object in the request - obj := h.defaulter.DeepCopyObject().(Defaulter) - if err := h.decoder.Decode(req, obj); err != nil { - return Errored(http.StatusBadRequest, err) - } - - // Default the object - obj.Default() - marshalled, err := json.Marshal(obj) - if err != nil { - return Errored(http.StatusInternalServerError, err) - } - - // Create the patch - return PatchResponseFromRaw(req.Object.Raw, marshalled) -} diff --git a/pkg/webhook/admission/defaulter_custom.go b/pkg/webhook/admission/defaulter_custom.go index 5f697e7dce..a703cbd2c5 100644 --- a/pkg/webhook/admission/defaulter_custom.go +++ b/pkg/webhook/admission/defaulter_custom.go @@ -21,11 +21,14 @@ import ( "encoding/json" "errors" "net/http" + "slices" + "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" ) // CustomDefaulter defines functions for setting defaults on resources. @@ -33,17 +36,41 @@ type CustomDefaulter interface { Default(ctx context.Context, obj runtime.Object) error } +type defaulterOptions struct { + removeUnknownOrOmitableFields bool +} + +// DefaulterOption defines the type of a CustomDefaulter's option +type DefaulterOption func(*defaulterOptions) + +// DefaulterRemoveUnknownOrOmitableFields makes the defaulter prune fields that are in the json object retrieved by the +// webhook but not in the local go type json representation. This happens for example when the CRD in the apiserver has +// fields that our go type doesn't know about, because it's outdated, or the field has a zero value and is `omitempty`. +func DefaulterRemoveUnknownOrOmitableFields(o *defaulterOptions) { + o.removeUnknownOrOmitableFields = true +} + // WithCustomDefaulter creates a new Webhook for a CustomDefaulter interface. -func WithCustomDefaulter(scheme *runtime.Scheme, obj runtime.Object, defaulter CustomDefaulter) *Webhook { +func WithCustomDefaulter(scheme *runtime.Scheme, obj runtime.Object, defaulter CustomDefaulter, opts ...DefaulterOption) *Webhook { + options := &defaulterOptions{} + for _, o := range opts { + o(options) + } return &Webhook{ - Handler: &defaulterForType{object: obj, defaulter: defaulter, decoder: NewDecoder(scheme)}, + Handler: &defaulterForType{ + object: obj, + defaulter: defaulter, + decoder: NewDecoder(scheme), + removeUnknownOrOmitableFields: options.removeUnknownOrOmitableFields, + }, } } type defaulterForType struct { - defaulter CustomDefaulter - object runtime.Object - decoder *Decoder + defaulter CustomDefaulter + object runtime.Object + decoder Decoder + removeUnknownOrOmitableFields bool } // Handle handles admission requests. @@ -76,6 +103,12 @@ func (h *defaulterForType) Handle(ctx context.Context, req Request) Response { return Errored(http.StatusBadRequest, err) } + // Keep a copy of the object if needed + var originalObj runtime.Object + if !h.removeUnknownOrOmitableFields { + originalObj = obj.DeepCopyObject() + } + // Default the object if err := h.defaulter.Default(ctx, obj); err != nil { var apiStatus apierrors.APIStatus @@ -90,5 +123,43 @@ func (h *defaulterForType) Handle(ctx context.Context, req Request) Response { if err != nil { return Errored(http.StatusInternalServerError, err) } - return PatchResponseFromRaw(req.Object.Raw, marshalled) + + handlerResponse := PatchResponseFromRaw(req.Object.Raw, marshalled) + if !h.removeUnknownOrOmitableFields { + handlerResponse = h.dropSchemeRemovals(handlerResponse, originalObj, req.Object.Raw) + } + return handlerResponse +} + +func (h *defaulterForType) dropSchemeRemovals(r Response, original runtime.Object, raw []byte) Response { + const opRemove = "remove" + if !r.Allowed || r.PatchType == nil { + return r + } + + // If we don't have removals in the patch. + if !slices.ContainsFunc(r.Patches, func(o jsonpatch.JsonPatchOperation) bool { return o.Operation == opRemove }) { + return r + } + + // Get the raw to original patch + marshalledOriginal, err := json.Marshal(original) + if err != nil { + return Errored(http.StatusInternalServerError, err) + } + + patchOriginal, err := jsonpatch.CreatePatch(raw, marshalledOriginal) + if err != nil { + return Errored(http.StatusInternalServerError, err) + } + removedByScheme := sets.New(slices.DeleteFunc(patchOriginal, func(p jsonpatch.JsonPatchOperation) bool { return p.Operation != opRemove })...) + + r.Patches = slices.DeleteFunc(r.Patches, func(p jsonpatch.JsonPatchOperation) bool { + return p.Operation == opRemove && removedByScheme.Has(p) + }) + + if len(r.Patches) == 0 { + r.PatchType = nil + } + return r } diff --git a/pkg/webhook/admission/defaulter_custom_test.go b/pkg/webhook/admission/defaulter_custom_test.go new file mode 100644 index 0000000000..1bc26e59f4 --- /dev/null +++ b/pkg/webhook/admission/defaulter_custom_test.go @@ -0,0 +1,169 @@ +/* +Copyright 2021 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "context" + "maps" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gomodules.xyz/jsonpatch/v2" + + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("Defaulter Handler", func() { + + It("should remove unknown fields when DefaulterRemoveUnknownFields is passed", func(ctx SpecContext) { + obj := &TestDefaulter{} + handler := WithCustomDefaulter(admissionScheme, obj, &TestCustomDefaulter{}, DefaulterRemoveUnknownOrOmitableFields) + + resp := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte(`{"newField":"foo", "totalReplicas":5}`), + }, + }, + }) + Expect(resp.Allowed).Should(BeTrue()) + Expect(resp.Patches).To(HaveLen(4)) + Expect(resp.Patches).To(ContainElements( + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/labels", + Value: map[string]any{"foo": "bar"}, + }, + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/replica", + Value: 2.0, + }, + jsonpatch.JsonPatchOperation{ + Operation: "remove", + Path: "/newField", + }, + jsonpatch.JsonPatchOperation{ + Operation: "remove", + Path: "/totalReplicas", + }, + )) + Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + + It("should preserve unknown fields by default", func(ctx SpecContext) { + obj := &TestDefaulter{} + handler := WithCustomDefaulter(admissionScheme, obj, &TestCustomDefaulter{}) + + resp := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, + Object: runtime.RawExtension{ + Raw: []byte(`{"newField":"foo", "totalReplicas":5}`), + }, + }, + }) + Expect(resp.Allowed).Should(BeTrue()) + Expect(resp.Patches).To(HaveLen(3)) + Expect(resp.Patches).To(ContainElements( + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/labels", + Value: map[string]any{"foo": "bar"}, + }, + jsonpatch.JsonPatchOperation{ + Operation: "add", + Path: "/replica", + Value: 2.0, + }, + jsonpatch.JsonPatchOperation{ + Operation: "remove", + Path: "/totalReplicas", + }, + )) + Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + + It("should return ok if received delete verb in defaulter handler", func(ctx SpecContext) { + obj := &TestDefaulter{} + handler := WithCustomDefaulter(admissionScheme, obj, &TestCustomDefaulter{}) + resp := handler.Handle(ctx, Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + }, + }, + }) + Expect(resp.Allowed).Should(BeTrue()) + Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) + }) +}) + +// TestDefaulter. +var _ runtime.Object = &TestDefaulter{} + +type TestDefaulter struct { + Labels map[string]string `json:"labels,omitempty"` + + Replica int `json:"replica,omitempty"` + TotalReplicas int `json:"totalReplicas,omitempty"` +} + +var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestDefaulter"} + +func (d *TestDefaulter) GetObjectKind() schema.ObjectKind { return d } +func (d *TestDefaulter) DeepCopyObject() runtime.Object { + return &TestDefaulter{ + Labels: maps.Clone(d.Labels), + Replica: d.Replica, + TotalReplicas: d.TotalReplicas, + } +} + +func (d *TestDefaulter) GroupVersionKind() schema.GroupVersionKind { + return testDefaulterGVK +} + +func (d *TestDefaulter) SetGroupVersionKind(gvk schema.GroupVersionKind) {} + +var _ runtime.Object = &TestDefaulterList{} + +type TestDefaulterList struct{} + +func (*TestDefaulterList) GetObjectKind() schema.ObjectKind { return nil } +func (*TestDefaulterList) DeepCopyObject() runtime.Object { return nil } + +// TestCustomDefaulter +type TestCustomDefaulter struct{} + +func (d *TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) error { + o := obj.(*TestDefaulter) + + if o.Labels == nil { + o.Labels = map[string]string{} + } + o.Labels["foo"] = "bar" + + if o.Replica < 2 { + o.Replica = 2 + } + o.TotalReplicas = 0 + return nil +} diff --git a/pkg/webhook/admission/defaulter_test.go b/pkg/webhook/admission/defaulter_test.go deleted file mode 100644 index cf7571663c..0000000000 --- a/pkg/webhook/admission/defaulter_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package admission - -import ( - "context" - "net/http" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - admissionv1 "k8s.io/api/admission/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -var _ = Describe("Defaulter Handler", func() { - - It("should return ok if received delete verb in defaulter handler", func() { - obj := &TestDefaulter{} - handler := DefaultingWebhookFor(admissionScheme, obj) - - resp := handler.Handle(context.TODO(), Request{ - AdmissionRequest: admissionv1.AdmissionRequest{ - Operation: admissionv1.Delete, - OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - }, - }, - }) - Expect(resp.Allowed).Should(BeTrue()) - Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) - }) - -}) - -// TestDefaulter. -var _ runtime.Object = &TestDefaulter{} - -type TestDefaulter struct { - Replica int `json:"replica,omitempty"` -} - -var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestDefaulter"} - -func (d *TestDefaulter) GetObjectKind() schema.ObjectKind { return d } -func (d *TestDefaulter) DeepCopyObject() runtime.Object { - return &TestDefaulter{ - Replica: d.Replica, - } -} - -func (d *TestDefaulter) GroupVersionKind() schema.GroupVersionKind { - return testDefaulterGVK -} - -func (d *TestDefaulter) SetGroupVersionKind(gvk schema.GroupVersionKind) {} - -var _ runtime.Object = &TestDefaulterList{} - -type TestDefaulterList struct{} - -func (*TestDefaulterList) GetObjectKind() schema.ObjectKind { return nil } -func (*TestDefaulterList) DeepCopyObject() runtime.Object { return nil } - -func (d *TestDefaulter) Default() { - if d.Replica < 2 { - d.Replica = 2 - } -} diff --git a/pkg/webhook/admission/http.go b/pkg/webhook/admission/http.go index 84ab5e75a4..f049fb66e6 100644 --- a/pkg/webhook/admission/http.go +++ b/pkg/webhook/admission/http.go @@ -34,6 +34,26 @@ import ( var admissionScheme = runtime.NewScheme() var admissionCodecs = serializer.NewCodecFactory(admissionScheme) +// adapted from https://github.com/kubernetes/kubernetes/blob/c28c2009181fcc44c5f6b47e10e62dacf53e4da0/staging/src/k8s.io/pod-security-admission/cmd/webhook/server/server.go +// +// From https://github.com/kubernetes/apiserver/blob/d6876a0600de06fef75968c4641c64d7da499f25/pkg/server/config.go#L433-L442C5: +// +// 1.5MB is the recommended client request size in byte +// the etcd server should accept. See +// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56. +// A request body might be encoded in json, and is converted to +// proto when persisted in etcd, so we allow 2x as the largest request +// body size to be accepted and decoded in a write request. +// +// For the admission request, we can infer that it contains at most two objects +// (the old and new versions of the object being admitted), each of which can +// be at most 3MB in size. For the rest of the request, we can assume that +// it will be less than 1MB in size. Therefore, we can set the max request +// size to 7MB. +// If your use case requires larger max request sizes, please +// open an issue (https://github.com/kubernetes-sigs/controller-runtime/issues/new). +const maxRequestSize = int64(7 * 1024 * 1024) + func init() { utilruntime.Must(v1.AddToScheme(admissionScheme)) utilruntime.Must(v1beta1.AddToScheme(admissionScheme)) @@ -42,27 +62,30 @@ func init() { var _ http.Handler = &Webhook{} func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var body []byte - var err error ctx := r.Context() if wh.WithContextFunc != nil { ctx = wh.WithContextFunc(ctx, r) } - var reviewResponse Response - if r.Body == nil { - err = errors.New("request body is empty") + if r.Body == nil || r.Body == http.NoBody { + err := errors.New("request body is empty") wh.getLogger(nil).Error(err, "bad request") - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) return } defer r.Body.Close() - if body, err = io.ReadAll(r.Body); err != nil { + limitedReader := &io.LimitedReader{R: r.Body, N: maxRequestSize} + body, err := io.ReadAll(limitedReader) + if err != nil { wh.getLogger(nil).Error(err, "unable to read the body from the incoming request") - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) + return + } + if limitedReader.N <= 0 { + err := fmt.Errorf("request entity is too large; limit is %d bytes", maxRequestSize) + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request; limit reached") + wh.writeResponse(w, Errored(http.StatusRequestEntityTooLarge, err)) return } @@ -70,8 +93,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { err = fmt.Errorf("contentType=%s, expected application/json", contentType) wh.getLogger(nil).Error(err, "unable to process a request with unknown content type") - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) return } @@ -89,14 +111,12 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, actualAdmRevGVK, err := admissionCodecs.UniversalDeserializer().Decode(body, nil, &ar) if err != nil { wh.getLogger(nil).Error(err, "unable to decode the request") - reviewResponse = Errored(http.StatusBadRequest, err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(http.StatusBadRequest, err)) return } - wh.getLogger(&req).V(4).Info("received request") + wh.getLogger(&req).V(5).Info("received request") - reviewResponse = wh.Handle(ctx, req) - wh.writeResponseTyped(w, reviewResponse, actualAdmRevGVK) + wh.writeResponseTyped(w, wh.Handle(ctx, req), actualAdmRevGVK) } // writeResponse writes response to w generically, i.e. without encoding GVK information. @@ -136,11 +156,11 @@ func (wh *Webhook) writeAdmissionResponse(w io.Writer, ar v1.AdmissionReview) { } } else { res := ar.Response - if log := wh.getLogger(nil); log.V(4).Enabled() { + if log := wh.getLogger(nil); log.V(5).Enabled() { if res.Result != nil { log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason, "message", res.Result.Message) } - log.V(4).Info("wrote response", "requestID", res.UID, "allowed", res.Allowed) + log.V(5).Info("wrote response", "requestID", res.UID, "allowed", res.Allowed) } } } diff --git a/pkg/webhook/admission/http_test.go b/pkg/webhook/admission/http_test.go index be10aea459..9cea9dd9e7 100644 --- a/pkg/webhook/admission/http_test.go +++ b/pkg/webhook/admission/http_test.go @@ -19,6 +19,7 @@ package admission import ( "bytes" "context" + "crypto/rand" "fmt" "io" "net/http" @@ -84,6 +85,32 @@ var _ = Describe("Admission Webhooks", func() { Expect(respRecorder.Body.String()).To(Equal(expected)) }) + It("should error when given a NoBody", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: http.NoBody, + } + + expected := `{"response":{"uid":"","allowed":false,"status":{"metadata":{},"message":"request body is empty","code":400}}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + + It("should error when given an infinite body", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: nopCloser{Reader: rand.Reader}, + } + + expected := `{"response":{"uid":"","allowed":false,"status":{"metadata":{},"message":"request entity is too large; limit is 7340032 bytes","code":413}}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + It("should return the response given by the handler with version defaulted to v1", func() { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, @@ -129,7 +156,7 @@ var _ = Describe("Admission Webhooks", func() { Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should present the Context from the HTTP request, if any", func() { + It("should present the Context from the HTTP request, if any", func(specCtx SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Body: nopCloser{Reader: bytes.NewBufferString(`{"request":{}}`)}, @@ -149,13 +176,13 @@ var _ = Describe("Admission Webhooks", func() { expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"message":%q,"code":200}}} `, gvkJSONv1, value) - ctx, cancel := context.WithCancel(context.WithValue(context.Background(), key, value)) + ctx, cancel := context.WithCancel(context.WithValue(specCtx, key, value)) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should mutate the Context from the HTTP request, if func supplied", func() { + It("should mutate the Context from the HTTP request, if func supplied", func(specCtx SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Body: nopCloser{Reader: bytes.NewBufferString(`{"request":{}}`)}, @@ -176,7 +203,7 @@ var _ = Describe("Admission Webhooks", func() { expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"message":%q,"code":200}}} `, gvkJSONv1, "application/json") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specCtx) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) diff --git a/pkg/webhook/admission/metrics/metrics.go b/pkg/webhook/admission/metrics/metrics.go new file mode 100644 index 0000000000..358a3a9162 --- /dev/null +++ b/pkg/webhook/admission/metrics/metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // WebhookPanics is a prometheus counter metrics which holds the total + // number of panics from webhooks. + WebhookPanics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "controller_runtime_webhook_panics_total", + Help: "Total number of webhook panics", + }, []string{}) +) + +func init() { + metrics.Registry.MustRegister( + WebhookPanics, + ) + // Init metric. + WebhookPanics.WithLabelValues().Add(0) +} diff --git a/pkg/webhook/admission/multi.go b/pkg/webhook/admission/multi.go index 2f7820d04b..ef9c456248 100644 --- a/pkg/webhook/admission/multi.go +++ b/pkg/webhook/admission/multi.go @@ -31,6 +31,7 @@ type multiMutating []Handler func (hs multiMutating) Handle(ctx context.Context, req Request) Response { patches := []jsonpatch.JsonPatchOperation{} + warnings := []string{} for _, handler := range hs { resp := handler.Handle(ctx, req) if !resp.Allowed { @@ -42,6 +43,7 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response { resp.PatchType, admissionv1.PatchTypeJSONPatch)) } patches = append(patches, resp.Patches...) + warnings = append(warnings, resp.Warnings...) } var err error marshaledPatch, err := json.Marshal(patches) @@ -55,6 +57,7 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response { Code: http.StatusOK, }, Patch: marshaledPatch, + Warnings: warnings, PatchType: func() *admissionv1.PatchType { pt := admissionv1.PatchTypeJSONPatch; return &pt }(), }, } @@ -71,11 +74,13 @@ func MultiMutatingHandler(handlers ...Handler) Handler { type multiValidating []Handler func (hs multiValidating) Handle(ctx context.Context, req Request) Response { + warnings := []string{} for _, handler := range hs { resp := handler.Handle(ctx, req) if !resp.Allowed { return resp } + warnings = append(warnings, resp.Warnings...) } return Response{ AdmissionResponse: admissionv1.AdmissionResponse{ @@ -83,6 +88,7 @@ func (hs multiValidating) Handle(ctx context.Context, req Request) Response { Result: &metav1.Status{ Code: http.StatusOK, }, + Warnings: warnings, }, } } diff --git a/pkg/webhook/admission/multi_test.go b/pkg/webhook/admission/multi_test.go index da85a52e42..888836ed67 100644 --- a/pkg/webhook/admission/multi_test.go +++ b/pkg/webhook/admission/multi_test.go @@ -46,23 +46,46 @@ var _ = Describe("Multi-Handler Admission Webhooks", func() { }, } + withWarnings := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + return Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Warnings: []string{"handler-warning"}, + }, + } + }, + } + Context("with validating handlers", func() { - It("should deny the request if any handler denies the request", func() { + It("should deny the request if any handler denies the request", func(ctx SpecContext) { By("setting up a handler with accept and deny") handler := MultiValidatingHandler(alwaysAllow, alwaysDeny) By("checking that the handler denies the request") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeFalse()) + Expect(resp.Warnings).To(BeEmpty()) }) - It("should allow the request if all handlers allow the request", func() { + It("should allow the request if all handlers allow the request", func(ctx SpecContext) { By("setting up a handler with only accept") handler := MultiValidatingHandler(alwaysAllow, alwaysAllow) By("checking that the handler allows the request") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeTrue()) + Expect(resp.Warnings).To(BeEmpty()) + }) + + It("should show the warnings if all handlers allow the request", func(ctx SpecContext) { + By("setting up a handler with only accept") + handler := MultiValidatingHandler(alwaysAllow, withWarnings) + + By("checking that the handler allows the request") + resp := handler.Handle(ctx, Request{}) + Expect(resp.Allowed).To(BeTrue()) + Expect(resp.Warnings).To(HaveLen(1)) }) }) @@ -107,26 +130,60 @@ var _ = Describe("Multi-Handler Admission Webhooks", func() { }, } - It("should not return any patches if the request is denied", func() { + patcher3 := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + return Response{ + Patches: []jsonpatch.JsonPatchOperation{ + { + Operation: "add", + Path: "/metadata/annotation/newest-key", + Value: "value", + }, + }, + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Warnings: []string{"annotation-warning"}, + PatchType: func() *admissionv1.PatchType { pt := admissionv1.PatchTypeJSONPatch; return &pt }(), + }, + } + }, + } + + It("should not return any patches if the request is denied", func(ctx SpecContext) { By("setting up a webhook with some patches and a deny") handler := MultiMutatingHandler(patcher1, patcher2, alwaysDeny) By("checking that the handler denies the request and produces no patches") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeFalse()) Expect(resp.Patches).To(BeEmpty()) }) - It("should produce all patches if the requests are all allowed", func() { + It("should produce all patches if the requests are all allowed", func(ctx SpecContext) { By("setting up a webhook with some patches") handler := MultiMutatingHandler(patcher1, patcher2, alwaysAllow) By("checking that the handler accepts the request and returns all patches") - resp := handler.Handle(context.Background(), Request{}) + resp := handler.Handle(ctx, Request{}) Expect(resp.Allowed).To(BeTrue()) Expect(resp.Patch).To(Equal([]byte( `[{"op":"add","path":"/metadata/annotation/new-key","value":"new-value"},` + `{"op":"replace","path":"/spec/replicas","value":"2"},{"op":"add","path":"/metadata/annotation/hello","value":"world"}]`))) }) + + It("should produce all patches if the requests are all allowed and show warnings", func(ctx SpecContext) { + By("setting up a webhook with some patches") + handler := MultiMutatingHandler(patcher1, patcher2, alwaysAllow, patcher3) + + By("checking that the handler accepts the request and returns all patches") + resp := handler.Handle(ctx, Request{}) + Expect(resp.Allowed).To(BeTrue()) + Expect(resp.Patch).To(Equal([]byte( + `[{"op":"add","path":"/metadata/annotation/new-key","value":"new-value"},` + + `{"op":"replace","path":"/spec/replicas","value":"2"},{"op":"add","path":"/metadata/annotation/hello","value":"world"},` + + `{"op":"add","path":"/metadata/annotation/newest-key","value":"value"}]`))) + Expect(resp.Warnings).To(HaveLen(1)) + }) + }) }) diff --git a/pkg/webhook/admission/validator.go b/pkg/webhook/admission/validator.go deleted file mode 100644 index 00bda8a4ce..0000000000 --- a/pkg/webhook/admission/validator.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -import ( - "context" - "errors" - "fmt" - "net/http" - - v1 "k8s.io/api/admission/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" -) - -// Warnings represents warning messages. -type Warnings []string - -// Validator defines functions for validating an operation. -// The custom resource kind which implements this interface can validate itself. -// To validate the custom resource with another specific struct, use CustomValidator instead. -type Validator interface { - runtime.Object - - // ValidateCreate validates the object on creation. - // The optional warnings will be added to the response as warning messages. - // Return an error if the object is invalid. - ValidateCreate() (warnings Warnings, err error) - - // ValidateUpdate validates the object on update. The oldObj is the object before the update. - // The optional warnings will be added to the response as warning messages. - // Return an error if the object is invalid. - ValidateUpdate(old runtime.Object) (warnings Warnings, err error) - - // ValidateDelete validates the object on deletion. - // The optional warnings will be added to the response as warning messages. - // Return an error if the object is invalid. - ValidateDelete() (warnings Warnings, err error) -} - -// ValidatingWebhookFor creates a new Webhook for validating the provided type. -func ValidatingWebhookFor(scheme *runtime.Scheme, validator Validator) *Webhook { - return &Webhook{ - Handler: &validatingHandler{validator: validator, decoder: NewDecoder(scheme)}, - } -} - -type validatingHandler struct { - validator Validator - decoder *Decoder -} - -// Handle handles admission requests. -func (h *validatingHandler) Handle(ctx context.Context, req Request) Response { - if h.decoder == nil { - panic("decoder should never be nil") - } - if h.validator == nil { - panic("validator should never be nil") - } - // Get the object in the request - obj := h.validator.DeepCopyObject().(Validator) - - var err error - var warnings []string - - switch req.Operation { - case v1.Connect: - // No validation for connect requests. - // TODO(vincepri): Should we validate CONNECT requests? In what cases? - case v1.Create: - if err = h.decoder.Decode(req, obj); err != nil { - return Errored(http.StatusBadRequest, err) - } - - warnings, err = obj.ValidateCreate() - case v1.Update: - oldObj := obj.DeepCopyObject() - - err = h.decoder.DecodeRaw(req.Object, obj) - if err != nil { - return Errored(http.StatusBadRequest, err) - } - err = h.decoder.DecodeRaw(req.OldObject, oldObj) - if err != nil { - return Errored(http.StatusBadRequest, err) - } - - warnings, err = obj.ValidateUpdate(oldObj) - case v1.Delete: - // In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346 - // OldObject contains the object being deleted - err = h.decoder.DecodeRaw(req.OldObject, obj) - if err != nil { - return Errored(http.StatusBadRequest, err) - } - - warnings, err = obj.ValidateDelete() - default: - return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation %q", req.Operation)) - } - - if err != nil { - var apiStatus apierrors.APIStatus - if errors.As(err, &apiStatus) { - return validationResponseFromStatus(false, apiStatus.Status()).WithWarnings(warnings...) - } - return Denied(err.Error()).WithWarnings(warnings...) - } - return Allowed("").WithWarnings(warnings...) -} diff --git a/pkg/webhook/admission/validator_custom.go b/pkg/webhook/admission/validator_custom.go index e99fbd8a85..ef1be52a8f 100644 --- a/pkg/webhook/admission/validator_custom.go +++ b/pkg/webhook/admission/validator_custom.go @@ -27,10 +27,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// Warnings represents warning messages. +type Warnings []string + // CustomValidator defines functions for validating an operation. // The object to be validated is passed into methods as a parameter. type CustomValidator interface { - // ValidateCreate validates the object on creation. // The optional warnings will be added to the response as warning messages. // Return an error if the object is invalid. @@ -57,7 +59,7 @@ func WithCustomValidator(scheme *runtime.Scheme, obj runtime.Object, validator C type validatorForType struct { validator CustomValidator object runtime.Object - decoder *Decoder + decoder Decoder } // Handle handles admission requests. diff --git a/pkg/webhook/admission/validator_test.go b/pkg/webhook/admission/validator_custom_test.go similarity index 77% rename from pkg/webhook/admission/validator_test.go rename to pkg/webhook/admission/validator_custom_test.go index 404fad9016..7c9615df71 100644 --- a/pkg/webhook/admission/validator_test.go +++ b/pkg/webhook/admission/validator_custom_test.go @@ -1,10 +1,8 @@ /* Copyright 2021 The Kubernetes Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software @@ -28,27 +26,25 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/kubernetes/scheme" ) var fakeValidatorVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "fakeValidator"} -var _ = Describe("validatingHandler", func() { - - decoder := NewDecoder(scheme.Scheme) +var _ = Describe("customValidatingHandler", func() { Context("when dealing with successful results without warning", func() { - f := &fakeValidator{ErrorToReturn: nil, GVKToReturn: fakeValidatorVK, WarningsToReturn: nil} - handler := validatingHandler{validator: f, decoder: decoder} + val := &fakeCustomValidator{ErrorToReturn: nil, GVKToReturn: fakeValidatorVK, WarningsToReturn: nil} + f := &fakeValidator{} + handler := WithCustomValidator(admissionScheme, f, val) - It("should return 200 in response when create succeeds", func() { + It("should return 200 in response when create succeeds", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Create, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -57,18 +53,18 @@ var _ = Describe("validatingHandler", func() { Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) }) - It("should return 200 in response when update succeeds", func() { + It("should return 200 in response when update succeeds", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -76,14 +72,14 @@ var _ = Describe("validatingHandler", func() { Expect(response.Result.Code).Should(Equal(int32(http.StatusOK))) }) - It("should return 200 in response when delete succeeds", func() { + It("should return 200 in response when delete succeeds", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Delete, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -95,19 +91,19 @@ var _ = Describe("validatingHandler", func() { const warningMessage = "warning message" const anotherWarningMessage = "another warning message" Context("when dealing with successful results with warning", func() { - f := &fakeValidator{ErrorToReturn: nil, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{ + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: nil, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{ warningMessage, anotherWarningMessage, }} - handler := validatingHandler{validator: f, decoder: decoder} - - It("should return 200 in response when create succeeds, with warning messages", func() { - response := handler.Handle(context.TODO(), Request{ + handler := WithCustomValidator(admissionScheme, f, val) + It("should return 200 in response when create succeeds, with warning messages", func(ctx SpecContext) { + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Create, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -118,18 +114,18 @@ var _ = Describe("validatingHandler", func() { Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) }) - It("should return 200 in response when update succeeds, with warning messages", func() { + It("should return 200 in response when update succeeds, with warning messages", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -139,14 +135,15 @@ var _ = Describe("validatingHandler", func() { Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) }) - It("should return 200 in response when delete succeeds, with warning messages", func() { + It("should return 200 in response when delete succeeds, with warning messages", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Delete, OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, + Raw: []byte("{}"), + + Object: f, }, }, }) @@ -165,17 +162,18 @@ var _ = Describe("validatingHandler", func() { Code: http.StatusUnprocessableEntity, }, } - f := &fakeValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{warningMessage, anotherWarningMessage}} - handler := validatingHandler{validator: f, decoder: decoder} + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{warningMessage, anotherWarningMessage}} + handler := WithCustomValidator(admissionScheme, f, val) - It("should propagate the Status from ValidateCreate's return value to the HTTP response", func() { + It("should propagate the Status from ValidateCreate's return value to the HTTP response", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Create, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -188,18 +186,19 @@ var _ = Describe("validatingHandler", func() { }) - It("should propagate the Status from ValidateUpdate's return value to the HTTP response", func() { + It("should propagate the Status from ValidateUpdate's return value to the HTTP response", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, OldObject: runtime.RawExtension{ - Raw: []byte("{}"), - Object: handler.validator, + Raw: []byte("{}"), + + Object: f, }, }, }) @@ -212,14 +211,15 @@ var _ = Describe("validatingHandler", func() { }) - It("should propagate the Status from ValidateDelete's return value to the HTTP response", func() { + It("should propagate the Status from ValidateDelete's return value to the HTTP response", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -242,17 +242,19 @@ var _ = Describe("validatingHandler", func() { Code: http.StatusUnprocessableEntity, }, } - f := &fakeValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: nil} - handler := validatingHandler{validator: f, decoder: decoder} + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: nil} + handler := WithCustomValidator(admissionScheme, f, val) - It("should propagate the Status from ValidateCreate's return value to the HTTP response", func() { + It("should propagate the Status from ValidateCreate's return value to the HTTP response", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ - response := handler.Handle(context.TODO(), Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Create, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -263,18 +265,18 @@ var _ = Describe("validatingHandler", func() { }) - It("should propagate the Status from ValidateUpdate's return value to the HTTP response", func() { + It("should propagate the Status from ValidateUpdate's return value to the HTTP response", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -285,20 +287,21 @@ var _ = Describe("validatingHandler", func() { }) - It("should propagate the Status from ValidateDelete's return value to the HTTP response", func() { + It("should propagate the Status from ValidateDelete's return value to the HTTP response", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Delete, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) Expect(response.Allowed).Should(BeFalse()) Expect(response.Result.Code).Should(Equal(expectedError.Status().Code)) + Expect(*response.Result).Should(Equal(expectedError.Status())) }) @@ -308,17 +311,19 @@ var _ = Describe("validatingHandler", func() { Context("when dealing with non-status errors, without warning messages", func() { expectedError := errors.New("some error") - f := &fakeValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK} - handler := validatingHandler{validator: f, decoder: decoder} + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK} + handler := WithCustomValidator(admissionScheme, f, val) + + It("should return 403 response when ValidateCreate with error message embedded", func(ctx SpecContext) { - It("should return 403 response when ValidateCreate with error message embedded", func() { + response := handler.Handle(ctx, Request{ - response := handler.Handle(context.TODO(), Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Create, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -329,18 +334,19 @@ var _ = Describe("validatingHandler", func() { }) - It("should return 403 response when ValidateUpdate returns non-APIStatus error", func() { + It("should return 403 response when ValidateUpdate returns non-APIStatus error", func(ctx SpecContext) { + + response := handler.Handle(ctx, Request{ - response := handler.Handle(context.TODO(), Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -351,13 +357,13 @@ var _ = Describe("validatingHandler", func() { }) - It("should return 403 response when ValidateDelete returns non-APIStatus error", func() { - response := handler.Handle(context.TODO(), Request{ + It("should return 403 response when ValidateDelete returns non-APIStatus error", func(ctx SpecContext) { + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Delete, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -371,21 +377,24 @@ var _ = Describe("validatingHandler", func() { Context("when dealing with non-status errors, with warning messages", func() { expectedError := errors.New("some error") - f := &fakeValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{warningMessage, anotherWarningMessage}} - handler := validatingHandler{validator: f, decoder: decoder} + f := &fakeValidator{} + val := &fakeCustomValidator{ErrorToReturn: expectedError, GVKToReturn: fakeValidatorVK, WarningsToReturn: []string{warningMessage, anotherWarningMessage}} + handler := WithCustomValidator(admissionScheme, f, val) - It("should return 403 response when ValidateCreate with error message embedded", func() { + It("should return 403 response when ValidateCreate with error message embedded", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Create, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) Expect(response.Allowed).Should(BeFalse()) + Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) Expect(response.Result.Message).Should(Equal(expectedError.Error())) @@ -393,18 +402,19 @@ var _ = Describe("validatingHandler", func() { Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) }) - It("should return 403 response when ValidateUpdate returns non-APIStatus error", func() { + It("should return 403 response when ValidateUpdate returns non-APIStatus error", func(ctx SpecContext) { - response := handler.Handle(context.TODO(), Request{ + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Update, Object: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, OldObject: runtime.RawExtension{ + Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -412,18 +422,19 @@ var _ = Describe("validatingHandler", func() { Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) Expect(response.Result.Message).Should(Equal(expectedError.Error())) + Expect(response.AdmissionResponse.Warnings).Should(ContainElement(warningMessage)) Expect(response.AdmissionResponse.Warnings).Should(ContainElement(anotherWarningMessage)) }) - It("should return 403 response when ValidateDelete returns non-APIStatus error", func() { - response := handler.Handle(context.TODO(), Request{ + It("should return 403 response when ValidateDelete returns non-APIStatus error", func(ctx SpecContext) { + response := handler.Handle(ctx, Request{ AdmissionRequest: admissionv1.AdmissionRequest{ Operation: admissionv1.Delete, OldObject: runtime.RawExtension{ Raw: []byte("{}"), - Object: handler.validator, + Object: f, }, }, }) @@ -447,12 +458,12 @@ var _ = Describe("validatingHandler", func() { }) -// fakeValidator provides fake validating webhook functionality for testing -// It implements the admission.Validator interface and +// fakeCustomValidator provides fake validating webhook functionality for testing +// It implements the admission.CustomValidator interface and // rejects all requests with the same configured error // or passes if ErrorToReturn is nil. // And it would always return configured warning messages WarningsToReturn. -type fakeValidator struct { +type fakeCustomValidator struct { // ErrorToReturn is the error for which the fakeValidator rejects all requests ErrorToReturn error `json:"errorToReturn,omitempty"` // GVKToReturn is the GroupVersionKind that the webhook operates on @@ -461,18 +472,23 @@ type fakeValidator struct { WarningsToReturn []string } -func (v *fakeValidator) ValidateCreate() (warnings Warnings, err error) { +func (v *fakeCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (warnings Warnings, err error) { return v.WarningsToReturn, v.ErrorToReturn } -func (v *fakeValidator) ValidateUpdate(old runtime.Object) (warnings Warnings, err error) { +func (v *fakeCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (warnings Warnings, err error) { return v.WarningsToReturn, v.ErrorToReturn } -func (v *fakeValidator) ValidateDelete() (warnings Warnings, err error) { +func (v *fakeCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (warnings Warnings, err error) { return v.WarningsToReturn, v.ErrorToReturn } +type fakeValidator struct { + // GVKToReturn is the GroupVersionKind that the webhook operates on + GVKToReturn schema.GroupVersionKind +} + func (v *fakeValidator) SetGroupVersionKind(gvk schema.GroupVersionKind) { v.GVKToReturn = gvk } @@ -487,8 +503,6 @@ func (v *fakeValidator) GetObjectKind() schema.ObjectKind { func (v *fakeValidator) DeepCopyObject() runtime.Object { return &fakeValidator{ - ErrorToReturn: v.ErrorToReturn, - GVKToReturn: v.GVKToReturn, - WarningsToReturn: v.WarningsToReturn, + GVKToReturn: v.GVKToReturn, } } diff --git a/pkg/webhook/admission/webhook.go b/pkg/webhook/admission/webhook.go index f1767f31b2..cba6da2cb0 100644 --- a/pkg/webhook/admission/webhook.go +++ b/pkg/webhook/admission/webhook.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/json" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/klog/v2" + admissionmetrics "sigs.k8s.io/controller-runtime/pkg/webhook/admission/metrics" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" @@ -123,7 +124,8 @@ type Webhook struct { Handler Handler // RecoverPanic indicates whether the panic caused by webhook should be recovered. - RecoverPanic bool + // Defaults to true. + RecoverPanic *bool // WithContextFunc will allow you to take the http.Request.Context() and // add any additional information such as passing the request path or @@ -141,8 +143,9 @@ type Webhook struct { } // WithRecoverPanic takes a bool flag which indicates whether the panic caused by webhook should be recovered. +// Defaults to true. func (wh *Webhook) WithRecoverPanic(recoverPanic bool) *Webhook { - wh.RecoverPanic = recoverPanic + wh.RecoverPanic = &recoverPanic return wh } @@ -151,17 +154,26 @@ func (wh *Webhook) WithRecoverPanic(recoverPanic bool) *Webhook { // If the webhook is validating type, it delegates the AdmissionRequest to each handler and // deny the request if anyone denies. func (wh *Webhook) Handle(ctx context.Context, req Request) (response Response) { - if wh.RecoverPanic { - defer func() { - if r := recover(); r != nil { + defer func() { + if r := recover(); r != nil { + admissionmetrics.WebhookPanics.WithLabelValues().Inc() + + if wh.RecoverPanic == nil || *wh.RecoverPanic { for _, fn := range utilruntime.PanicHandlers { - fn(r) + fn(ctx, r) } response = Errored(http.StatusInternalServerError, fmt.Errorf("panic: %v [recovered]", r)) + // Note: We explicitly have to set the response UID. Usually that is done via resp.Complete below, + // but if we encounter a panic in wh.Handler.Handle we are never going to reach resp.Complete. + response.UID = req.UID return } - }() - } + + log := logf.FromContext(ctx) + log.Info(fmt.Sprintf("Observed a panic in webhook: %v", r)) + panic(r) + } + }() reqLog := wh.getLogger(&req) ctx = logf.IntoContext(ctx, reqLog) @@ -169,7 +181,10 @@ func (wh *Webhook) Handle(ctx context.Context, req Request) (response Response) resp := wh.Handler.Handle(ctx, req) if err := resp.Complete(req); err != nil { reqLog.Error(err, "unable to encode response") - return Errored(http.StatusInternalServerError, errUnableToEncodeResponse) + resp := Errored(http.StatusInternalServerError, errUnableToEncodeResponse) + // Note: We explicitly have to set the response UID. Usually that is done via resp.Complete. + resp.UID = req.UID + return resp } return resp diff --git a/pkg/webhook/admission/webhook_test.go b/pkg/webhook/admission/webhook_test.go index c7fc3b09ac..5176077368 100644 --- a/pkg/webhook/admission/webhook_test.go +++ b/pkg/webhook/admission/webhook_test.go @@ -30,6 +30,7 @@ import ( authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" machinerytypes "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -63,40 +64,40 @@ var _ = Describe("Admission Webhooks", func() { return webhook } - It("should invoke the handler to get a response", func() { + It("should invoke the handler to get a response", func(ctx SpecContext) { By("setting up a webhook with an allow handler") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that it allowed the request") Expect(resp.Allowed).To(BeTrue()) }) - It("should ensure that the response's UID is set to the request's UID", func() { + It("should ensure that the response's UID is set to the request's UID", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{AdmissionRequest: admissionv1.AdmissionRequest{UID: "foobar"}}) + resp := webhook.Handle(ctx, Request{AdmissionRequest: admissionv1.AdmissionRequest{UID: "foobar"}}) By("checking that the response share's the request's UID") Expect(resp.UID).To(Equal(machinerytypes.UID("foobar"))) }) - It("should populate the status on a response if one is not provided", func() { + It("should populate the status on a response if one is not provided", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the response share's the request's UID") Expect(resp.Result).To(Equal(&metav1.Status{Code: http.StatusOK})) }) - It("shouldn't overwrite the status on a response", func() { + It("shouldn't overwrite the status on a response", func(ctx SpecContext) { By("setting up a webhook that sets a status") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { @@ -110,14 +111,14 @@ var _ = Describe("Admission Webhooks", func() { } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the message is intact") Expect(resp.Result).NotTo(BeNil()) Expect(resp.Result.Message).To(Equal("Ground Control to Major Tom")) }) - It("should serialize patch operations into a single jsonpatch blob", func() { + It("should serialize patch operations into a single jsonpatch blob", func(ctx SpecContext) { By("setting up a webhook with a patching handler") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { @@ -126,7 +127,7 @@ var _ = Describe("Admission Webhooks", func() { } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that a JSON patch is populated on the response") patchType := admissionv1.PatchTypeJSONPatch @@ -134,7 +135,7 @@ var _ = Describe("Admission Webhooks", func() { Expect(resp.Patch).To(Equal([]byte(`[{"op":"add","path":"/a","value":2},{"op":"replace","path":"/b","value":4}]`))) }) - It("should pass a request logger via the context", func() { + It("should pass a request logger via the context", func(ctx SpecContext) { By("setting up a webhook that uses the request logger") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { @@ -150,7 +151,7 @@ var _ = Describe("Admission Webhooks", func() { } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{AdmissionRequest: admissionv1.AdmissionRequest{ + resp := webhook.Handle(ctx, Request{AdmissionRequest: admissionv1.AdmissionRequest{ UID: "test123", Name: "foo", Namespace: "bar", @@ -169,7 +170,7 @@ var _ = Describe("Admission Webhooks", func() { Eventually(logBuffer).Should(gbytes.Say(`"msg":"Received request","object":{"name":"foo","namespace":"bar"},"namespace":"bar","name":"foo","resource":{"group":"apps","version":"v1","resource":"deployments"},"user":"tim","requestID":"test123"}`)) }) - It("should pass a request logger created by LogConstructor via the context", func() { + It("should pass a request logger created by LogConstructor via the context", func(ctx SpecContext) { By("setting up a webhook that uses the request logger") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { @@ -188,7 +189,7 @@ var _ = Describe("Admission Webhooks", func() { } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{AdmissionRequest: admissionv1.AdmissionRequest{ + resp := webhook.Handle(ctx, Request{AdmissionRequest: admissionv1.AdmissionRequest{ UID: "test123", Operation: admissionv1.Create, }}) @@ -199,7 +200,34 @@ var _ = Describe("Admission Webhooks", func() { }) Describe("panic recovery", func() { - It("should recover panic if RecoverPanic is true", func() { + It("should recover panic if RecoverPanic is true by default", func(ctx SpecContext) { + panicHandler := func() *Webhook { + handler := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + panic("fake panic test") + }, + } + webhook := &Webhook{ + Handler: handler, + // RecoverPanic defaults to true. + } + + return webhook + } + + By("setting up a webhook with a panicking handler") + webhook := panicHandler() + + By("invoking the webhook") + resp := webhook.Handle(ctx, Request{}) + + By("checking that it errored the request") + Expect(resp.Allowed).To(BeFalse()) + Expect(resp.Result.Code).To(Equal(int32(http.StatusInternalServerError))) + Expect(resp.Result.Message).To(Equal("panic: fake panic test [recovered]")) + }) + + It("should recover panic if RecoverPanic is true", func(ctx SpecContext) { panicHandler := func() *Webhook { handler := &fakeHandler{ fn: func(ctx context.Context, req Request) Response { @@ -208,7 +236,7 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: handler, - RecoverPanic: true, + RecoverPanic: ptr.To[bool](true), } return webhook @@ -218,7 +246,7 @@ var _ = Describe("Admission Webhooks", func() { webhook := panicHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that it errored the request") Expect(resp.Allowed).To(BeFalse()) @@ -226,7 +254,7 @@ var _ = Describe("Admission Webhooks", func() { Expect(resp.Result.Message).To(Equal("panic: fake panic test [recovered]")) }) - It("should not recover panic if RecoverPanic is false by default", func() { + It("should not recover panic if RecoverPanic is false", func(ctx SpecContext) { panicHandler := func() *Webhook { handler := &fakeHandler{ fn: func(ctx context.Context, req Request) Response { @@ -234,7 +262,8 @@ var _ = Describe("Admission Webhooks", func() { }, } webhook := &Webhook{ - Handler: handler, + Handler: handler, + RecoverPanic: ptr.To[bool](false), } return webhook @@ -247,20 +276,19 @@ var _ = Describe("Admission Webhooks", func() { webhook := panicHandler() By("invoking the webhook") - webhook.Handle(context.Background(), Request{}) + webhook.Handle(ctx, Request{}) }) }) }) -var _ = Describe("Should be able to write/read admission.Request to/from context", func() { - ctx := context.Background() +var _ = It("Should be able to write/read admission.Request to/from context", func(specContext SpecContext) { testRequest := Request{ admissionv1.AdmissionRequest{ UID: "test-uid", }, } - ctx = NewContextWithRequest(ctx, testRequest) + ctx := NewContextWithRequest(specContext, testRequest) gotRequest, err := RequestFromContext(ctx) Expect(err).To(Not(HaveOccurred())) diff --git a/pkg/webhook/alias.go b/pkg/webhook/alias.go index 293137db49..2882e7bab3 100644 --- a/pkg/webhook/alias.go +++ b/pkg/webhook/alias.go @@ -23,12 +23,6 @@ import ( // define some aliases for common bits of the webhook functionality -// Defaulter defines functions for setting defaults on resources. -type Defaulter = admission.Defaulter - -// Validator defines functions for validating an operation. -type Validator = admission.Validator - // CustomDefaulter defines functions for setting defaults on resources. type CustomDefaulter = admission.CustomDefaulter diff --git a/pkg/webhook/authentication/http.go b/pkg/webhook/authentication/http.go index 51bc93ee19..abf95e5421 100644 --- a/pkg/webhook/authentication/http.go +++ b/pkg/webhook/authentication/http.go @@ -34,6 +34,13 @@ import ( var authenticationScheme = runtime.NewScheme() var authenticationCodecs = serializer.NewCodecFactory(authenticationScheme) +// The TokenReview resource mostly contains a bearer token which +// at most should have a few KB's of size, so we picked 1 MB to +// have plenty of buffer. +// If your use case requires larger max request sizes, please +// open an issue (https://github.com/kubernetes-sigs/controller-runtime/issues/new). +const maxRequestSize = int64(1 * 1024 * 1024) + func init() { utilruntime.Must(authenticationv1.AddToScheme(authenticationScheme)) utilruntime.Must(authenticationv1beta1.AddToScheme(authenticationScheme)) @@ -42,36 +49,38 @@ func init() { var _ http.Handler = &Webhook{} func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { - var body []byte - var err error ctx := r.Context() if wh.WithContextFunc != nil { ctx = wh.WithContextFunc(ctx, r) } - var reviewResponse Response - if r.Body == nil { - err = errors.New("request body is empty") + if r.Body == nil || r.Body == http.NoBody { + err := errors.New("request body is empty") wh.getLogger(nil).Error(err, "bad request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(err)) return } defer r.Body.Close() - if body, err = io.ReadAll(r.Body); err != nil { + limitedReader := &io.LimitedReader{R: r.Body, N: maxRequestSize} + body, err := io.ReadAll(limitedReader) + if err != nil { wh.getLogger(nil).Error(err, "unable to read the body from the incoming request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(err)) + return + } + if limitedReader.N <= 0 { + err := fmt.Errorf("request entity is too large; limit is %d bytes", maxRequestSize) + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request; limit reached") + wh.writeResponse(w, Errored(err)) return } // verify the content type is accurate if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { - err = fmt.Errorf("contentType=%s, expected application/json", contentType) + err := fmt.Errorf("contentType=%s, expected application/json", contentType) wh.getLogger(nil).Error(err, "unable to process a request with unknown content type") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(err)) return } @@ -90,22 +99,19 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, actualTokRevGVK, err := authenticationCodecs.UniversalDeserializer().Decode(body, nil, &ar) if err != nil { wh.getLogger(nil).Error(err, "unable to decode the request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(err)) return } - wh.getLogger(&req).V(4).Info("received request") + wh.getLogger(&req).V(5).Info("received request") if req.Spec.Token == "" { - err = errors.New("token is empty") + err := errors.New("token is empty") wh.getLogger(&req).Error(err, "bad request") - reviewResponse = Errored(err) - wh.writeResponse(w, reviewResponse) + wh.writeResponse(w, Errored(err)) return } - reviewResponse = wh.Handle(ctx, req) - wh.writeResponseTyped(w, reviewResponse, actualTokRevGVK) + wh.writeResponseTyped(w, wh.Handle(ctx, req), actualTokRevGVK) } // writeResponse writes response to w generically, i.e. without encoding GVK information. @@ -135,7 +141,7 @@ func (wh *Webhook) writeTokenResponse(w io.Writer, ar authenticationv1.TokenRevi wh.writeResponse(w, Errored(err)) } res := ar - wh.getLogger(nil).V(4).Info("wrote response", "requestID", res.UID, "authenticated", res.Status.Authenticated) + wh.getLogger(nil).V(5).Info("wrote response", "requestID", res.UID, "authenticated", res.Status.Authenticated) } // unversionedTokenReview is used to decode both v1 and v1beta1 TokenReview types. diff --git a/pkg/webhook/authentication/http_test.go b/pkg/webhook/authentication/http_test.go index 86bd5d0153..e51b2af7e6 100644 --- a/pkg/webhook/authentication/http_test.go +++ b/pkg/webhook/authentication/http_test.go @@ -19,6 +19,7 @@ package authentication import ( "bytes" "context" + "crypto/rand" "fmt" "io" "net/http" @@ -49,10 +50,10 @@ var _ = Describe("Authentication Webhooks", func() { It("should return bad-request when given an empty body", func() { req := &http.Request{Body: nil} - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"request body is empty"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"request body is empty"}} ` webhook.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(expected)) + Expect(respRecorder.Body.String()).To(BeComparableTo(expected)) }) It("should return bad-request when given the wrong content-type", func() { @@ -62,7 +63,7 @@ var _ = Describe("Authentication Webhooks", func() { Body: nopCloser{Reader: bytes.NewBuffer(nil)}, } - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"contentType=application/foo, expected application/json"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"contentType=application/foo, expected application/json"}} ` webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) @@ -75,7 +76,7 @@ var _ = Describe("Authentication Webhooks", func() { Body: nopCloser{Reader: bytes.NewBufferString("{")}, } - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"couldn't get version/kind; json parse error: unexpected end of JSON input"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"couldn't get version/kind; json parse error: unexpected end of JSON input"}} ` webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) @@ -88,7 +89,33 @@ var _ = Describe("Authentication Webhooks", func() { Body: nopCloser{Reader: bytes.NewBufferString(`{"spec":{"token":""}}`)}, } - expected := `{"metadata":{"creationTimestamp":null},"spec":{},"status":{"user":{},"error":"token is empty"}} + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"token is empty"}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + + It("should error when given a NoBody", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: http.NoBody, + } + + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"request body is empty"}} +` + webhook.ServeHTTP(respRecorder, req) + Expect(respRecorder.Body.String()).To(Equal(expected)) + }) + + It("should error when given an infinite body", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Method: http.MethodPost, + Body: nopCloser{Reader: rand.Reader}, + } + + expected := `{"metadata":{},"spec":{},"status":{"user":{},"error":"request entity is too large; limit is 1048576 bytes"}} ` webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) @@ -104,7 +131,7 @@ var _ = Describe("Authentication Webhooks", func() { Handler: &fakeHandler{}, } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{}}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{}}} `, gvkJSONv1) webhook.ServeHTTP(respRecorder, req) @@ -121,13 +148,13 @@ var _ = Describe("Authentication Webhooks", func() { Handler: &fakeHandler{}, } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{}}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{}}} `, gvkJSONv1) webhook.ServeHTTP(respRecorder, req) Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should present the Context from the HTTP request, if any", func() { + It("should present the Context from the HTTP request, if any", func(specContext SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Method: http.MethodPost, @@ -145,16 +172,16 @@ var _ = Describe("Authentication Webhooks", func() { }, } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} `, gvkJSONv1, value) - ctx, cancel := context.WithCancel(context.WithValue(context.Background(), key, value)) + ctx, cancel := context.WithCancel(context.WithValue(specContext, key, value)) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should mutate the Context from the HTTP request, if func supplied", func() { + It("should mutate the Context from the HTTP request, if func supplied", func(specContext SpecContext) { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, Method: http.MethodPost, @@ -173,10 +200,10 @@ var _ = Describe("Authentication Webhooks", func() { }, } - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} + expected := fmt.Sprintf(`{%s,"metadata":{},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} `, gvkJSONv1, "application/json") - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(specContext) cancel() webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) diff --git a/pkg/webhook/authentication/webhook_test.go b/pkg/webhook/authentication/webhook_test.go index 3df446d898..22c4e284cd 100644 --- a/pkg/webhook/authentication/webhook_test.go +++ b/pkg/webhook/authentication/webhook_test.go @@ -47,40 +47,40 @@ var _ = Describe("Authentication Webhooks", func() { return webhook } - It("should invoke the handler to get a response", func() { + It("should invoke the handler to get a response", func(ctx SpecContext) { By("setting up a webhook with an allow handler") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that it allowed the request") Expect(resp.Status.Authenticated).To(BeTrue()) }) - It("should ensure that the response's UID is set to the request's UID", func() { + It("should ensure that the response's UID is set to the request's UID", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{TokenReview: authenticationv1.TokenReview{ObjectMeta: metav1.ObjectMeta{UID: "foobar"}}}) + resp := webhook.Handle(ctx, Request{TokenReview: authenticationv1.TokenReview{ObjectMeta: metav1.ObjectMeta{UID: "foobar"}}}) By("checking that the response share's the request's UID") Expect(resp.UID).To(Equal(machinerytypes.UID("foobar"))) }) - It("should populate the status on a response if one is not provided", func() { + It("should populate the status on a response if one is not provided", func(ctx SpecContext) { By("setting up a webhook") webhook := allowHandler() By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the response share's the request's UID") Expect(resp.Status).To(Equal(authenticationv1.TokenReviewStatus{Authenticated: true})) }) - It("shouldn't overwrite the status on a response", func() { + It("shouldn't overwrite the status on a response", func(ctx SpecContext) { By("setting up a webhook that sets a status") webhook := &Webhook{ Handler: HandlerFunc(func(ctx context.Context, req Request) Response { @@ -96,7 +96,7 @@ var _ = Describe("Authentication Webhooks", func() { } By("invoking the webhook") - resp := webhook.Handle(context.Background(), Request{}) + resp := webhook.Handle(ctx, Request{}) By("checking that the message is intact") Expect(resp.Status).NotTo(BeNil()) diff --git a/pkg/webhook/conversion/conversion.go b/pkg/webhook/conversion/conversion.go index 249a364b38..a26fa348bb 100644 --- a/pkg/webhook/conversion/conversion.go +++ b/pkg/webhook/conversion/conversion.go @@ -22,7 +22,9 @@ See pkg/conversion for interface definitions required to ensure an API Type is c package conversion import ( + "context" "encoding/json" + "errors" "fmt" "net/http" @@ -31,8 +33,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "sigs.k8s.io/controller-runtime/pkg/conversion" logf "sigs.k8s.io/controller-runtime/pkg/log" + conversionmetrics "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/metrics" ) var ( @@ -53,6 +57,8 @@ type webhook struct { var _ http.Handler = &webhook{} func (wh *webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + convertReview := &apix.ConversionReview{} err := json.NewDecoder(r.Body).Decode(convertReview) if err != nil { @@ -69,7 +75,7 @@ func (wh *webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { // TODO(droot): may be move the conversion logic to a separate module to // decouple it from the http layer ? - resp, err := wh.handleConvertRequest(convertReview.Request) + resp, err := wh.handleConvertRequest(ctx, convertReview.Request) if err != nil { log.Error(err, "failed to convert", "request", convertReview.Request.UID) convertReview.Response = errored(err) @@ -87,7 +93,18 @@ func (wh *webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // handles a version conversion request. -func (wh *webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) { +func (wh *webhook) handleConvertRequest(ctx context.Context, req *apix.ConversionRequest) (_ *apix.ConversionResponse, retErr error) { + defer func() { + if r := recover(); r != nil { + conversionmetrics.WebhookPanics.WithLabelValues().Inc() + + for _, fn := range utilruntime.PanicHandlers { + fn(ctx, r) + } + retErr = errors.New("internal error occurred during conversion") + return + } + }() if req == nil { return nil, fmt.Errorf("conversion request is nil") } diff --git a/pkg/webhook/conversion/conversion_test.go b/pkg/webhook/conversion/conversion_test.go index be984e232b..489689bccb 100644 --- a/pkg/webhook/conversion/conversion_test.go +++ b/pkg/webhook/conversion/conversion_test.go @@ -296,6 +296,29 @@ var _ = Describe("Conversion Webhook", func() { Expect(convReview.Response.ConvertedObjects).To(BeEmpty()) }) + It("should return error on panic in conversion", func() { + + v1Obj := makeV1Obj() + v1Obj.Spec.PanicInConversion = true + + convReq := &apix.ConversionReview{ + TypeMeta: metav1.TypeMeta{}, + Request: &apix.ConversionRequest{ + DesiredAPIVersion: "jobs.testprojects.kb.io/v3", + Objects: []runtime.RawExtension{ + { + Object: v1Obj, + }, + }, + }, + } + + convReview := doRequest(convReq) + + Expect(convReview.Response.ConvertedObjects).To(HaveLen(0)) + Expect(convReview.Response.Result.Status).To(Equal(metav1.StatusFailure)) + Expect(convReview.Response.Result.Message).To(Equal("internal error occurred during conversion")) + }) }) var _ = Describe("IsConvertible", func() { diff --git a/pkg/webhook/conversion/metrics/metrics.go b/pkg/webhook/conversion/metrics/metrics.go new file mode 100644 index 0000000000..c825f17f0b --- /dev/null +++ b/pkg/webhook/conversion/metrics/metrics.go @@ -0,0 +1,39 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // WebhookPanics is a prometheus counter metrics which holds the total + // number of panics from conversion webhooks. + WebhookPanics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "controller_runtime_conversion_webhook_panics_total", + Help: "Total number of conversion webhook panics", + }, []string{}) +) + +func init() { + metrics.Registry.MustRegister( + WebhookPanics, + ) + // Init metric. + WebhookPanics.WithLabelValues().Add(0) +} diff --git a/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go b/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go index bf99e2a204..c6065e1fb4 100644 --- a/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go +++ b/pkg/webhook/conversion/testdata/api/v1/externaljob_types.go @@ -17,6 +17,7 @@ package v1 import ( "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -31,6 +32,9 @@ type ExternalJobSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file RunAt string `json:"runAt"` + + // PanicInConversion triggers a panic during conversion when set to true. + PanicInConversion bool `json:"panicInConversion"` } // ExternalJobStatus defines the observed state of ExternalJob @@ -66,6 +70,9 @@ func init() { // ConvertTo implements conversion logic to convert to Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := dst.(type) { case *v2.ExternalJob: jobv2 := dst.(*v2.ExternalJob) @@ -80,6 +87,9 @@ func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { // ConvertFrom implements conversion logic to convert from Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertFrom(src conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := src.(type) { case *v2.ExternalJob: jobv2 := src.(*v2.ExternalJob) diff --git a/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go b/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go index 7208ba8c69..af7396abf1 100644 --- a/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go +++ b/pkg/webhook/conversion/testdata/api/v1/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -// +build !ignore_autogenerated +//go:build !ignore_autogenerated /* @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// autogenerated by controller-gen object, do not modify manually +// Code generated by controller-gen. DO NOT EDIT. package v1 @@ -54,7 +54,7 @@ func (in *ExternalJob) DeepCopyObject() runtime.Object { func (in *ExternalJobList) DeepCopyInto(out *ExternalJobList) { *out = *in out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ExternalJob, len(*in)) diff --git a/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go b/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go index de5a03a212..1f87e8a017 100644 --- a/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go +++ b/pkg/webhook/conversion/testdata/api/v2/externaljob_types.go @@ -27,6 +27,9 @@ type ExternalJobSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file ScheduleAt string `json:"scheduleAt"` + + // PanicInConversion triggers a panic during conversion when set to true. + PanicInConversion bool `json:"panicInConversion"` } // ExternalJobStatus defines the observed state of ExternalJob diff --git a/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go b/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go index 53c9f758b1..d5efd6150e 100644 --- a/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go +++ b/pkg/webhook/conversion/testdata/api/v2/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -// +build !ignore_autogenerated +//go:build !ignore_autogenerated /* @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// autogenerated by controller-gen object, do not modify manually +// Code generated by controller-gen. DO NOT EDIT. package v2 @@ -54,7 +54,7 @@ func (in *ExternalJob) DeepCopyObject() runtime.Object { func (in *ExternalJobList) DeepCopyInto(out *ExternalJobList) { *out = *in out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ExternalJob, len(*in)) diff --git a/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go b/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go index 15c438f68a..85a166b7cf 100644 --- a/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go +++ b/pkg/webhook/conversion/testdata/api/v3/externaljob_types.go @@ -17,6 +17,7 @@ package v3 import ( "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -31,6 +32,9 @@ type ExternalJobSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file DeferredAt string `json:"deferredAt"` + + // PanicInConversion triggers a panic during conversion when set to true. + PanicInConversion bool `json:"panicInConversion"` } // ExternalJobStatus defines the observed state of ExternalJob @@ -66,6 +70,9 @@ func init() { // ConvertTo implements conversion logic to convert to Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := dst.(type) { case *v2.ExternalJob: jobv2 := dst.(*v2.ExternalJob) @@ -80,6 +87,9 @@ func (ej *ExternalJob) ConvertTo(dst conversion.Hub) error { // ConvertFrom implements conversion logic to convert from Hub type (v2.ExternalJob // in this case) func (ej *ExternalJob) ConvertFrom(src conversion.Hub) error { + if ej.Spec.PanicInConversion { + panic("PanicInConversion field set to true") + } switch t := src.(type) { case *v2.ExternalJob: jobv2 := src.(*v2.ExternalJob) diff --git a/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go b/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go index a90942b427..d12b6910dc 100644 --- a/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go +++ b/pkg/webhook/conversion/testdata/api/v3/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -// +build !ignore_autogenerated +//go:build !ignore_autogenerated /* @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// autogenerated by controller-gen object, do not modify manually +// Code generated by controller-gen. DO NOT EDIT. package v3 @@ -54,7 +54,7 @@ func (in *ExternalJob) DeepCopyObject() runtime.Object { func (in *ExternalJobList) DeepCopyInto(out *ExternalJobList) { *out = *in out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) if in.Items != nil { in, out := &in.Items, &out.Items *out = make([]ExternalJob, len(*in)) diff --git a/pkg/webhook/conversion/testdata/main.go b/pkg/webhook/conversion/testdata/main.go index a3922da009..ea6b8275a8 100644 --- a/pkg/webhook/conversion/testdata/main.go +++ b/pkg/webhook/conversion/testdata/main.go @@ -25,6 +25,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" jobsv1 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v1" jobsv2 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v2" jobsv3 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v3" @@ -55,9 +56,9 @@ func main() { ctrl.SetLogger(zap.Logger(true)) mgr, err := ctrl.NewManager(context.Background(), ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: metricsAddr, - LeaderElection: enableLeaderElection, + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: metricsAddr}, + LeaderElection: enableLeaderElection, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/pkg/webhook/example_test.go b/pkg/webhook/example_test.go index f68008755d..7c4f718f4c 100644 --- a/pkg/webhook/example_test.go +++ b/pkg/webhook/example_test.go @@ -145,7 +145,7 @@ func ExampleStandaloneWebhook() { mux.Handle("/validating", validatingHookHandler) // Run your handler - if err := http.ListenAndServe(port, mux); err != nil { //nolint:gosec // it's fine to not set timeouts here + if err := http.ListenAndServe(port, mux); err != nil { panic(err) } } diff --git a/pkg/webhook/internal/metrics/metrics.go b/pkg/webhook/internal/metrics/metrics.go index 557004908b..f1e6ce68f5 100644 --- a/pkg/webhook/internal/metrics/metrics.go +++ b/pkg/webhook/internal/metrics/metrics.go @@ -18,6 +18,7 @@ package metrics import ( "net/http" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -30,8 +31,11 @@ var ( // of processing admission requests. RequestLatency = prometheus.NewHistogramVec( prometheus.HistogramOpts{ - Name: "controller_runtime_webhook_latency_seconds", - Help: "Histogram of the latency of processing admission requests", + Name: "controller_runtime_webhook_latency_seconds", + Help: "Histogram of the latency of processing admission requests", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, }, []string{"webhook"}, ) diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index 23d5bf4350..4d8ae9ec7a 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -77,37 +77,33 @@ type Options struct { // It will be defaulted to 9443 if unspecified. Port int - // CertDir is the directory that contains the server key and certificate. The - // server key and certificate. + // CertDir is the directory that contains the server key and certificate. Defaults to + // /k8s-webhook-server/serving-certs. CertDir string // CertName is the server certificate name. Defaults to tls.crt. // - // Note: This option should only be set when TLSOpts does not override GetCertificate. + // Note: This option is only used when TLSOpts does not set GetCertificate. CertName string // KeyName is the server key name. Defaults to tls.key. // - // Note: This option should only be set when TLSOpts does not override GetCertificate. + // Note: This option is only used when TLSOpts does not set GetCertificate. KeyName string // ClientCAName is the CA certificate name which server used to verify remote(client)'s certificate. // Defaults to "", which means server does not verify client's certificate. ClientCAName string - // TLSVersion is the minimum version of TLS supported. Accepts - // "", "1.0", "1.1", "1.2" and "1.3" only ("" is equivalent to "1.0" for backwards compatibility) - // Deprecated: Use TLSOpts instead. - TLSMinVersion string - - // TLSOpts is used to allow configuring the TLS config used for the server + // TLSOpts is used to allow configuring the TLS config used for the server. + // This also allows providing a certificate via GetCertificate. TLSOpts []func(*tls.Config) // WebhookMux is the multiplexer that handles different webhooks. WebhookMux *http.ServeMux } -// NewServer constructs a new Server from the provided options. +// NewServer constructs a new webhook.Server from the provided options. func NewServer(o Options) Server { return &DefaultServer{ Options: o, @@ -187,42 +183,15 @@ func (s *DefaultServer) Register(path string, hook http.Handler) { regLog.Info("Registering webhook") } -// tlsVersion converts from human-readable TLS version (for example "1.1") -// to the values accepted by tls.Config (for example 0x301). -func tlsVersion(version string) (uint16, error) { - switch version { - // default is previous behaviour - case "": - return tls.VersionTLS10, nil - case "1.0": - return tls.VersionTLS10, nil - case "1.1": - return tls.VersionTLS11, nil - case "1.2": - return tls.VersionTLS12, nil - case "1.3": - return tls.VersionTLS13, nil - default: - return 0, fmt.Errorf("invalid TLSMinVersion %v: expects 1.0, 1.1, 1.2, 1.3 or empty", version) - } -} - // Start runs the server. // It will install the webhook related resources depend on the server configuration. func (s *DefaultServer) Start(ctx context.Context) error { s.defaultingOnce.Do(s.setDefaults) - baseHookLog := log.WithName("webhooks") - baseHookLog.Info("Starting webhook server") - - tlsMinVersion, err := tlsVersion(s.Options.TLSMinVersion) - if err != nil { - return err - } + log.Info("Starting webhook server") - cfg := &tls.Config{ //nolint:gosec + cfg := &tls.Config{ NextProtos: []string{"h2"}, - MinVersion: tlsMinVersion, } // fallback TLS config ready, will now mutate if passer wants full control over it for _, op := range s.Options.TLSOpts { @@ -303,7 +272,7 @@ func (s *DefaultServer) Start(ctx context.Context) error { // server has been started. func (s *DefaultServer) StartedChecker() healthz.Checker { config := &tls.Config{ - InsecureSkipVerify: true, //nolint:gosec // config is used to connect to our own webhook port. + InsecureSkipVerify: true, } return func(req *http.Request) error { s.mu.Lock() diff --git a/pkg/webhook/server_test.go b/pkg/webhook/server_test.go index 9702b0fd6e..6542222585 100644 --- a/pkg/webhook/server_test.go +++ b/pkg/webhook/server_test.go @@ -29,23 +29,26 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/webhook" ) var _ = Describe("Webhook Server", func() { var ( - ctx context.Context - ctxCancel context.CancelFunc - testHostPort string - client *http.Client - server webhook.Server - servingOpts envtest.WebhookInstallOptions + ctxCancel context.CancelFunc + testHostPort string + client *http.Client + server webhook.Server + servingOpts envtest.WebhookInstallOptions + genericStartServer func(f func(ctx context.Context)) (done <-chan struct{}) ) BeforeEach(func() { - ctx, ctxCancel = context.WithCancel(context.Background()) - // closed in individual tests differently + var ctx context.Context + // Has to be derived from context.Background() as it needs to be + // valid past the BeforeEach + ctx, ctxCancel = context.WithCancel(context.Background()) //nolint:forbidigo servingOpts = envtest.WebhookInstallOptions{} Expect(servingOpts.PrepWithoutInstalling()).To(Succeed()) @@ -66,27 +69,27 @@ var _ = Describe("Webhook Server", func() { Port: servingOpts.LocalServingPort, CertDir: servingOpts.LocalServingCertDir, }) + + genericStartServer = func(f func(ctx context.Context)) (done <-chan struct{}) { + doneCh := make(chan struct{}) + go func() { + defer GinkgoRecover() + defer close(doneCh) + f(ctx) + }() + // wait till we can ping the server to start the test + Eventually(func() error { + _, err := client.Get(fmt.Sprintf("https://%s/unservedpath", testHostPort)) + return err + }).Should(Succeed()) + + return doneCh + } }) AfterEach(func() { Expect(servingOpts.Cleanup()).To(Succeed()) }) - genericStartServer := func(f func(ctx context.Context)) (done <-chan struct{}) { - doneCh := make(chan struct{}) - go func() { - defer GinkgoRecover() - defer close(doneCh) - f(ctx) - }() - // wait till we can ping the server to start the test - Eventually(func() error { - _, err := client.Get(fmt.Sprintf("https://%s/unservedpath", testHostPort)) - return err - }).Should(Succeed()) - - return doneCh - } - startServer := func() (done <-chan struct{}) { return genericStartServer(func(ctx context.Context) { Expect(server.Start(ctx)).To(Succeed()) @@ -169,14 +172,14 @@ var _ = Describe("Webhook Server", func() { tls.TLS_AES_128_GCM_SHA256, tls.TLS_AES_256_GCM_SHA384, } + cfg.MinVersion = tls.VersionTLS12 // save cfg after changes to test against finalCfg = cfg } server = webhook.NewServer(webhook.Options{ - Host: servingOpts.LocalServingHost, - Port: servingOpts.LocalServingPort, - CertDir: servingOpts.LocalServingCertDir, - TLSMinVersion: "1.2", + Host: servingOpts.LocalServingHost, + Port: servingOpts.LocalServingPort, + CertDir: servingOpts.LocalServingCertDir, TLSOpts: []func(*tls.Config){ tlsCfgFunc, }, @@ -213,13 +216,14 @@ var _ = Describe("Webhook Server", func() { return &finalCert, nil } server = &webhook.DefaultServer{Options: webhook.Options{ - Host: servingOpts.LocalServingHost, - Port: servingOpts.LocalServingPort, - CertDir: servingOpts.LocalServingCertDir, - TLSMinVersion: "1.2", + Host: servingOpts.LocalServingHost, + Port: servingOpts.LocalServingPort, + CertDir: servingOpts.LocalServingCertDir, + TLSOpts: []func(*tls.Config){ func(cfg *tls.Config) { cfg.GetCertificate = finalGetCertificate + cfg.MinVersion = tls.VersionTLS12 // save cfg after changes to test against finalCfg = cfg }, diff --git a/pkg/webhook/webhook_integration_test.go b/pkg/webhook/webhook_integration_test.go index 716692124b..cbb5b711f7 100644 --- a/pkg/webhook/webhook_integration_test.go +++ b/pkg/webhook/webhook_integration_test.go @@ -28,6 +28,8 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -71,58 +73,57 @@ var _ = Describe("Webhook", func() { } }) Context("when running a webhook server with a manager", func() { - It("should reject create request for webhook that rejects all requests", func() { + It("should reject create request for webhook that rejects all requests", func(ctx SpecContext) { m, err := manager.New(cfg, manager.Options{ - Port: testenv.WebhookInstallOptions.LocalServingPort, - Host: testenv.WebhookInstallOptions.LocalServingHost, - CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, - TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: testenv.WebhookInstallOptions.LocalServingPort, + Host: testenv.WebhookInstallOptions.LocalServingHost, + CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + }), }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{d: admission.NewDecoder(testenv.Scheme)}}) - ctx, cancel := context.WithCancel(context.Background()) go func() { err := server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { - err := c.Create(context.TODO(), obj) + err := c.Create(ctx, obj) return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) - - cancel() }) - It("should reject create request for multi-webhook that rejects all requests", func() { + It("should reject create request for multi-webhook that rejects all requests", func(ctx SpecContext) { m, err := manager.New(cfg, manager.Options{ - MetricsBindAddress: "0", - Port: testenv.WebhookInstallOptions.LocalServingPort, - Host: testenv.WebhookInstallOptions.LocalServingHost, - CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, - TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + Metrics: metricsserver.Options{BindAddress: "0"}, + WebhookServer: webhook.NewServer(webhook.Options{ + Port: testenv.WebhookInstallOptions.LocalServingPort, + Host: testenv.WebhookInstallOptions.LocalServingHost, + CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, + }), }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() server.Register("/failing", &webhook.Admission{Handler: admission.MultiValidatingHandler(&rejectingValidator{d: admission.NewDecoder(testenv.Scheme)})}) - ctx, cancel := context.WithCancel(context.Background()) go func() { + defer GinkgoRecover() err = server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { - err = c.Create(context.TODO(), obj) + err = c.Create(ctx, obj) return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) - - cancel() }) }) Context("when running a webhook server without a manager", func() { - It("should reject create request for webhook that rejects all requests", func() { + It("should reject create request for webhook that rejects all requests", func(ctx SpecContext) { server := webhook.NewServer(webhook.Options{ Port: testenv.WebhookInstallOptions.LocalServingPort, Host: testenv.WebhookInstallOptions.LocalServingHost, @@ -130,24 +131,21 @@ var _ = Describe("Webhook", func() { }) server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{d: admission.NewDecoder(testenv.Scheme)}}) - ctx, cancel := context.WithCancel(context.Background()) go func() { err := server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { - err := c.Create(context.TODO(), obj) + err := c.Create(ctx, obj) return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) - - cancel() }) }) }) type rejectingValidator struct { - d *admission.Decoder + d admission.Decoder } func (v *rejectingValidator) Handle(ctx context.Context, req admission.Request) admission.Response { diff --git a/tools/setup-envtest/README.md b/tools/setup-envtest/README.md index 40379c9b8c..a4de6f3eae 100644 --- a/tools/setup-envtest/README.md +++ b/tools/setup-envtest/README.md @@ -4,13 +4,19 @@ This is a small tool that manages binaries for envtest. It can be used to download new binaries, list currently installed and available ones, and clean up versions. -To use it, just go-install it on 1.16+ (it's a separate, self-contained +To use it, just go-install it with Golang 1.24+ (it's a separate, self-contained module): ```shell go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ``` +If you are using Golang 1.23, use the `release-0.20` branch instead: + +```shell +go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.20 +``` + For full documentation, run it with the `--help` flag, but here are some examples: @@ -40,27 +46,33 @@ setup-envtest use -i --use-env # sideload a pre-downloaded tarball as Kubernetes 1.16.2 into our store setup-envtest sideload 1.16.2 < downloaded-envtest.tar.gz + +# Per default envtest binaries are downloaded from: +# https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml +# To download from a custom index use the following: +setup-envtest use --index https://custom.com/envtest-releases.yaml + ``` ## Where does it put all those binaries? By default, binaries are stored in a subdirectory of an OS-specific data -directory, as per the OS's conventions. +directory, as per the OS's conventions. On Linux, this is `$XDG_DATA_HOME`; on Windows, `%LocalAppData`; and on OSX, `~/Library/Application Support`. There's an overall folder that holds all files, and inside that is -a folder for each version/platform pair. The exact directory structure is -not guarnateed, except that the leaf directory will contain the names -expected by envtest. You should always use `setup-envtest fetch` or +a folder for each version/platform pair. The exact directory structure is +not guaranteed, except that the leaf directory will contain the names +expected by envtest. You should always use `setup-envtest fetch` or `setup-envtest switch` (generally with the `-p path` or `-p env` flags) to get the directory that you should use. ## Why do I have to do that `source <(blah blah blah)` thing This is a normal binary, not a shell script, so we can't set the parent -process's environment variables. If you use this by hand a lot and want +process's environment variables. If you use this by hand a lot and want to save the typing, you could put something like the following in your `~/.zshrc` (or similar for bash/fish/whatever, modified to those): @@ -79,7 +91,7 @@ setup-envtest() { There are a few options. First, you'll probably want to set the `-i/--installed` flag. If you want -to avoid forgetting to set this flag, set the `ENVTEST_INSTALLED_ONLY` +to avoid forgetting to set this flag, set the `ENVTEST_INSTALLED_ONLY` env variable, which will switch that flag on by default. Then, you have a few options for managing your binaries: @@ -90,36 +102,18 @@ Then, you have a few options for managing your binaries: `--use-env` makes the command unconditionally use the value of KUBEBUILDER_ASSETS as long as it contains the required binaries, and - `-i` indicates that we only ever want to work with installed binaries - (no reaching out the remote GCS storage). + `-i` indicates that we only ever want to work with installed binaries. As noted about, you can use `ENVTEST_INSTALLED_ONLY=true` to switch `-i` on by default, and you can use `ENVTEST_USE_ENV=true` to switch `--use-env` on by default. - If you want to use this tool, but download your gziped tarballs - separately, you can use the `sideload` command. You'll need to use the + separately, you can use the `sideload` command. You'll need to use the `-k/--version` flag to indicate which version you're sideloading. After that, it'll be as if you'd installed the binaries with `use`. -- If you want to talk to some internal source, you can use the - `--remote-bucket` and `--remote-server` options. The former sets which - GCS bucket to download from, and the latter sets the host to talk to as - if it were a GCS endpoint. Theoretically, you could use the latter - version to run an internal "mirror" -- the tool expects - - - `HOST/storage/v1/b/BUCKET/o` to return JSON like - - ```json - {"items": [ - {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}, - {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}, - ]} - ``` - - - `HOST/storage/v1/b/BUCKET/o/TARBALL_NAME` to return JSON like - `{"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}` - - - `HOST/storage/v1/b/BUCKET/o/TARBALL_NAME?alt=media` to return the - actual file contents +- If you want to talk to some internal source via HTTP, you can simply set `--index` + The index must contain references to envtest binary archives in the same format as: + https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml diff --git a/tools/setup-envtest/env/env.go b/tools/setup-envtest/env/env.go index e12a107352..6168739eb6 100644 --- a/tools/setup-envtest/env/env.go +++ b/tools/setup-envtest/env/env.go @@ -26,24 +26,24 @@ import ( // envtest binaries. // // In general, the methods will use the Exit{,Cause} functions from this package -// to indicate errors. Catch them with a `defer HandleExitWithCode()`. +// to indicate errors. Catch them with a `defer HandleExitWithCode()`. type Env struct { // the following *must* be set on input // Platform is our current platform Platform versions.PlatformItem - // VerifiySum indicates whether or not we should run checksums. + // VerifySum indicates whether we should run checksums. VerifySum bool - // NoDownload forces us to not contact GCS, looking only - // at local files instead. + // NoDownload forces us to not contact remote services, + // looking only at local files instead. NoDownload bool // ForceDownload forces us to ignore local files and always - // contact GCS & re-download. + // contact remote services & re-download. ForceDownload bool - // Client is our remote client for contacting GCS. - Client *remote.Client + // Client is our remote client for contacting remote services. + Client remote.Client // Log allows us to log. Log logr.Logger @@ -133,7 +133,7 @@ func (e *Env) ListVersions(ctx context.Context) { } // LatestVersion returns the latest version matching our version selector and -// platform from the remote server, with the correspoding checksum for later +// platform from the remote server, with the corresponding checksum for later // use as well. func (e *Env) LatestVersion(ctx context.Context) (versions.Concrete, versions.PlatformItem) { vers, err := e.Client.ListVersions(ctx) @@ -193,7 +193,7 @@ func (e *Env) ExistsAndValid() bool { // // If necessary, it will enumerate on-disk and remote versions to accomplish // this, finding a version that matches our version selector and platform. -// It will always yield a concrete version, it *may* yield a concrete platorm +// It will always yield a concrete version, it *may* yield a concrete platform // as well. func (e *Env) EnsureVersionIsSet(ctx context.Context) { if e.Version.AsConcrete() != nil { @@ -247,13 +247,13 @@ func (e *Env) EnsureVersionIsSet(ctx context.Context) { // if we're not forcing a download, and we have a newer local version, just use that if !e.ForceDownload && localVer != nil && localVer.NewerThan(serverVer) { - e.Platform.Platform = localPlat // update our data with md5 + e.Platform.Platform = localPlat // update our data with hash e.Version.MakeConcrete(*localVer) return } // otherwise, use the new version from the server - e.Platform = platform // update our data with md5 + e.Platform = platform // update our data with hash e.Version.MakeConcrete(serverVer) } @@ -266,13 +266,13 @@ func (e *Env) Fetch(ctx context.Context) { log := e.Log.WithName("fetch") // if we didn't just fetch it, grab the sum to verify - if e.VerifySum && e.Platform.MD5 == "" { + if e.VerifySum && e.Platform.Hash == nil { if err := e.Client.FetchSum(ctx, *e.Version.AsConcrete(), &e.Platform); err != nil { - ExitCause(2, err, "unable to fetch checksum for requested version") + ExitCause(2, err, "unable to fetch hash for requested version") } } if !e.VerifySum { - e.Platform.MD5 = "" // skip verification + e.Platform.Hash = nil // skip verification } var packedPath string @@ -365,8 +365,8 @@ func (e *Env) PrintInfo(printFmt PrintFormat) { case PrintOverview: fmt.Fprintf(e.Out, "Version: %s\n", e.Version) fmt.Fprintf(e.Out, "OS/Arch: %s\n", e.Platform) - if e.Platform.MD5 != "" { - fmt.Fprintf(e.Out, "md5: %s\n", e.Platform.MD5) + if e.Platform.Hash != nil { + fmt.Fprintf(e.Out, "%s: %s\n", e.Platform.Hash.Type, e.Platform.Hash.Value) } fmt.Fprintf(e.Out, "Path: %s\n", path) case PrintPath: @@ -409,7 +409,7 @@ func (e *Env) Sideload(ctx context.Context, input io.Reader) { } var ( - // expectedExectuables are the executables that are checked in PathMatches + // expectedExecutables are the executables that are checked in PathMatches // for non-store paths. expectedExecutables = []string{ "kube-apiserver", @@ -458,7 +458,7 @@ func (e *Env) PathMatches(value string) bool { } // versionFromPathName checks if the given path's last component looks like one -// of our versions, and, if so, what version it represents. If succesfull, +// of our versions, and, if so, what version it represents. If successful, // it'll set version and platform, and return true. Otherwise it returns // false. func (e *Env) versionFromPathName(value string) bool { diff --git a/tools/setup-envtest/go.mod b/tools/setup-envtest/go.mod index 542759927b..15c64f8b57 100644 --- a/tools/setup-envtest/go.mod +++ b/tools/setup-envtest/go.mod @@ -1,24 +1,29 @@ module sigs.k8s.io/controller-runtime/tools/setup-envtest -go 1.20 +go 1.24.0 require ( - github.com/go-logr/logr v1.2.0 - github.com/go-logr/zapr v1.2.0 - github.com/onsi/ginkgo/v2 v2.1.4 - github.com/onsi/gomega v1.19.0 - github.com/spf13/afero v1.6.0 - github.com/spf13/pflag v1.0.5 - go.uber.org/zap v1.19.1 + github.com/go-logr/logr v1.4.2 + github.com/go-logr/zapr v1.3.0 + github.com/onsi/ginkgo/v2 v2.22.2 + github.com/onsi/gomega v1.36.2 + github.com/spf13/afero v1.12.0 + github.com/spf13/pflag v1.0.6 + go.uber.org/zap v1.27.0 + k8s.io/apimachinery v0.34.1 + sigs.k8s.io/yaml v1.6.0 ) require ( - github.com/golang/protobuf v1.5.2 // indirect - go.uber.org/atomic v1.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect - golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect - golang.org/x/text v0.3.7 // indirect - google.golang.org/protobuf v1.26.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/tools v0.28.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/setup-envtest/go.sum b/tools/setup-envtest/go.sum index cb4c52f3e6..dfc8e7cce2 100644 --- a/tools/setup-envtest/go.sum +++ b/tools/setup-envtest/go.sum @@ -1,98 +1,52 @@ -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= -github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= +k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/tools/setup-envtest/main.go b/tools/setup-envtest/main.go index 8dca774157..7eb5ec43d3 100644 --- a/tools/setup-envtest/main.go +++ b/tools/setup-envtest/main.go @@ -49,10 +49,8 @@ var ( binDir = flag.String("bin-dir", "", "directory to store binary assets (default: $OS_SPECIFIC_DATA_DIR/envtest-binaries)") - remoteBucket = flag.String("remote-bucket", "kubebuilder-tools", "remote GCS bucket to download from") - remoteServer = flag.String("remote-server", "storage.googleapis.com", - "remote server to query from. You can override this if you want to run "+ - "an internal storage server instead, or for testing.") + + index = flag.String("index", remote.DefaultIndexURL, "index to discover envtest binaries") ) // TODO(directxman12): handle interrupts? @@ -81,13 +79,15 @@ func setupEnv(globalLog logr.Logger, version string) *envp.Env { } log.V(1).Info("using binaries directory", "dir", *binDir) + client := &remote.HTTPClient{ + Log: globalLog.WithName("storage-client"), + IndexURL: *index, + } + log.V(1).Info("using HTTP client", "index", *index) + env := &envp.Env{ - Log: globalLog, - Client: &remote.Client{ - Log: globalLog.WithName("storage-client"), - Bucket: *remoteBucket, - Server: *remoteServer, - }, + Log: globalLog, + Client: client, VerifySum: *verify, ForceDownload: *force, NoDownload: *installedOnly, @@ -169,7 +169,7 @@ Commands: use: get information for the requested version, downloading it if necessary and allowed. - Needs a concrete platform (no wildcards), but wilcard versions are supported. + Needs a concrete platform (no wildcards), but wildcard versions are supported. list: list installed *and* available versions matching the given version & platform. @@ -184,6 +184,9 @@ Commands: reads a .tar.gz file from stdin and expand it into the store. must have a concrete version and platform. + version: + list the installed version of setup-envtest. + Versions: Versions take the form of a small subset of semver selectors. @@ -256,7 +259,6 @@ Environment Variables: version = flag.Arg(1) } env := setupEnv(globalLog, version) - // perform our main set of actions switch action := flag.Arg(0); action { case "use": @@ -274,6 +276,8 @@ Environment Variables: Input: os.Stdin, PrintFormat: printFormat, }.Do(env) + case "version": + workflows.Version{}.Do(env) default: flag.Usage() envp.Exit(2, "unknown action %q", action) diff --git a/tools/setup-envtest/remote/client.go b/tools/setup-envtest/remote/client.go index be82532583..24efd6daff 100644 --- a/tools/setup-envtest/remote/client.go +++ b/tools/setup-envtest/remote/client.go @@ -1,219 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 -// Copyright 2021 The Kubernetes Authors +// Copyright 2024 The Kubernetes Authors package remote import ( "context" - "crypto/md5" //nolint:gosec - "encoding/base64" - "encoding/json" - "errors" - "fmt" "io" - "net/http" - "net/url" - "path" - "sort" - "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" ) -// objectList is the parts we need of the GCS "list-objects-in-bucket" endpoint. -type objectList struct { - Items []bucketObject `json:"items"` - NextPageToken string `json:"nextPageToken"` -} - -// bucketObject is the parts we need of the GCS object metadata. -type bucketObject struct { - Name string `json:"name"` - Hash string `json:"md5Hash"` -} - -// Client is a basic client for fetching versions of the envtest binary archives -// from GCS. -type Client struct { - // Bucket is the bucket to fetch from. - Bucket string - - // Server is the GCS-like storage server - Server string - - // Log allows us to log. - Log logr.Logger - - // Insecure uses http for testing - Insecure bool -} - -func (c *Client) scheme() string { - if c.Insecure { - return "http" - } - return "https" -} - -// ListVersions lists all available tools versions in the given bucket, along -// with supported os/arch combos and the corresponding hash. -// -// The results are sorted with newer versions first. -func (c *Client) ListVersions(ctx context.Context) ([]versions.Set, error) { - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o"), - } - query := make(url.Values) - - knownVersions := map[versions.Concrete][]versions.PlatformItem{} - for cont := true; cont; { - c.Log.V(1).Info("listing bucket to get versions", "bucket", c.Bucket) - - loc.RawQuery = query.Encode() - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return nil, fmt.Errorf("unable to construct request to list bucket items: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("unable to perform request to list bucket items: %w", err) - } - - err = func() error { - defer resp.Body.Close() - if resp.StatusCode != 200 { - return fmt.Errorf("unable list bucket items -- got status %q from GCS", resp.Status) - } - - var list objectList - if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { - return fmt.Errorf("unable unmarshal bucket items list: %w", err) - } - - // continue listing if needed - cont = list.NextPageToken != "" - query.Set("pageToken", list.NextPageToken) - - for _, item := range list.Items { - ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, item.Name) - if ver == nil { - c.Log.V(1).Info("skipping bucket object -- does not appear to be a versioned tools object", "name", item.Name) - continue - } - c.Log.V(1).Info("found version", "version", ver, "platform", details) - knownVersions[*ver] = append(knownVersions[*ver], versions.PlatformItem{ - Platform: details, - MD5: item.Hash, - }) - } - - return nil - }() - if err != nil { - return nil, err - } - } - - res := make([]versions.Set, 0, len(knownVersions)) - for ver, details := range knownVersions { - res = append(res, versions.Set{Version: ver, Platforms: details}) - } - // sort in inverse order so that the newest one is first - sort.Slice(res, func(i, j int) bool { - first, second := res[i].Version, res[j].Version - return first.NewerThan(second) - }) - - return res, nil -} - -// GetVersion downloads the given concrete version for the given concrete platform, writing it to the out. -func (c *Client) GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error { - itemName := platform.ArchiveName(version) - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), - RawQuery: "alt=media", - } - - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return fmt.Errorf("unable to construct request to fetch %s: %w", itemName, err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("unable to fetch %s (%s): %w", itemName, req.URL, err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("unable fetch %s (%s) -- got status %q from GCS", itemName, req.URL, resp.Status) - } - - if platform.MD5 != "" { - // stream in chunks to do the checksum, don't load the whole thing into - // memory to avoid causing issues with big files. - buf := make([]byte, 32*1024) // 32KiB, same as io.Copy - checksum := md5.New() //nolint:gosec - for cont := true; cont; { - amt, err := resp.Body.Read(buf) - if err != nil && !errors.Is(err, io.EOF) { - return fmt.Errorf("unable read next chunk of %s: %w", itemName, err) - } - if amt > 0 { - // checksum never returns errors according to docs - checksum.Write(buf[:amt]) - if _, err := out.Write(buf[:amt]); err != nil { - return fmt.Errorf("unable write next chunk of %s: %w", itemName, err) - } - } - cont = amt > 0 && !errors.Is(err, io.EOF) - } - - sum := base64.StdEncoding.EncodeToString(checksum.Sum(nil)) - - if sum != platform.MD5 { - return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (reported from GCS)", itemName, sum, platform.MD5) - } - } else if _, err := io.Copy(out, resp.Body); err != nil { - return fmt.Errorf("unable to download %s: %w", itemName, err) - } - return nil -} - -// FetchSum fetches the checksum for the given concrete version & platform into -// the given platform item. -func (c *Client) FetchSum(ctx context.Context, ver versions.Concrete, pl *versions.PlatformItem) error { - itemName := pl.ArchiveName(ver) - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), - } - - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return fmt.Errorf("unable to construct request to fetch metadata for %s: %w", itemName, err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("unable to fetch metadata for %s: %w", itemName, err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("unable fetch metadata for %s -- got status %q from GCS", itemName, resp.Status) - } +// Client is an interface to get and list envtest binary archives. +type Client interface { + ListVersions(ctx context.Context) ([]versions.Set, error) - var item bucketObject - if err := json.NewDecoder(resp.Body).Decode(&item); err != nil { - return fmt.Errorf("unable to unmarshal metadata for %s: %w", itemName, err) - } + GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error - pl.MD5 = item.Hash - return nil + FetchSum(ctx context.Context, ver versions.Concrete, pl *versions.PlatformItem) error } diff --git a/tools/setup-envtest/remote/http_client.go b/tools/setup-envtest/remote/http_client.go new file mode 100644 index 0000000000..0339654a82 --- /dev/null +++ b/tools/setup-envtest/remote/http_client.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 The Kubernetes Authors + +package remote + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "sort" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" + "sigs.k8s.io/yaml" +) + +// DefaultIndexURL is the default index used in HTTPClient. +var DefaultIndexURL = "https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml" + +var _ Client = &HTTPClient{} + +// HTTPClient is a client for fetching versions of the envtest binary archives +// from an index via HTTP. +type HTTPClient struct { + // Log allows us to log. + Log logr.Logger + + // IndexURL is the URL of the index, defaults to DefaultIndexURL. + IndexURL string +} + +// Index represents an index of envtest binary archives. Example: +// +// releases: +// v1.28.0: +// envtest-v1.28.0-darwin-amd64.tar.gz: +// hash: +// selfLink: +type Index struct { + // Releases maps Kubernetes versions to Releases (envtest archives). + Releases map[string]Release `json:"releases"` +} + +// Release maps an archive name to an archive. +type Release map[string]Archive + +// Archive contains the self link to an archive and its hash. +type Archive struct { + Hash string `json:"hash"` + SelfLink string `json:"selfLink"` +} + +// ListVersions lists all available tools versions in the index, along +// with supported os/arch combos and the corresponding hash. +// +// The results are sorted with newer versions first. +func (c *HTTPClient) ListVersions(ctx context.Context) ([]versions.Set, error) { + index, err := c.getIndex(ctx) + if err != nil { + return nil, err + } + + knownVersions := map[versions.Concrete][]versions.PlatformItem{} + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + c.Log.V(1).Info("found version", "version", ver, "platform", details) + knownVersions[*ver] = append(knownVersions[*ver], versions.PlatformItem{ + Platform: details, + Hash: &versions.Hash{ + Type: versions.SHA512HashType, + Encoding: versions.HexHashEncoding, + Value: archive.Hash, + }, + }) + } + } + + res := make([]versions.Set, 0, len(knownVersions)) + for ver, details := range knownVersions { + res = append(res, versions.Set{Version: ver, Platforms: details}) + } + // sort in inverse order so that the newest one is first + sort.Slice(res, func(i, j int) bool { + first, second := res[i].Version, res[j].Version + return first.NewerThan(second) + }) + + return res, nil +} + +// GetVersion downloads the given concrete version for the given concrete platform, writing it to the out. +func (c *HTTPClient) GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error { + index, err := c.getIndex(ctx) + if err != nil { + return err + } + + var loc *url.URL + var name string + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + + if *ver == version && details.OS == platform.OS && details.Arch == platform.Arch { + loc, err = url.Parse(archive.SelfLink) + if err != nil { + return fmt.Errorf("error parsing selfLink %q, %w", loc, err) + } + name = archiveName + break + } + } + } + if name == "" { + return fmt.Errorf("unable to find archive for %s (%s,%s)", version, platform.OS, platform.Arch) + } + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return fmt.Errorf("unable to construct request to fetch %s: %w", name, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("unable to fetch %s (%s): %w", name, req.URL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("unable fetch %s (%s) -- got status %q", name, req.URL, resp.Status) + } + + return readBody(resp, out, name, platform) +} + +// FetchSum fetches the checksum for the given concrete version & platform into +// the given platform item. +func (c *HTTPClient) FetchSum(ctx context.Context, version versions.Concrete, platform *versions.PlatformItem) error { + index, err := c.getIndex(ctx) + if err != nil { + return err + } + + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + + if *ver == version && details.OS == platform.OS && details.Arch == platform.Arch { + platform.Hash = &versions.Hash{ + Type: versions.SHA512HashType, + Encoding: versions.HexHashEncoding, + Value: archive.Hash, + } + return nil + } + } + } + + return fmt.Errorf("unable to find archive for %s (%s,%s)", version, platform.OS, platform.Arch) +} + +func (c *HTTPClient) getIndex(ctx context.Context) (*Index, error) { + indexURL := c.IndexURL + if indexURL == "" { + indexURL = DefaultIndexURL + } + + loc, err := url.Parse(indexURL) + if err != nil { + return nil, fmt.Errorf("unable to parse index URL: %w", err) + } + + c.Log.V(1).Info("listing versions", "index", indexURL) + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return nil, fmt.Errorf("unable to construct request to get index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("unable to perform request to get index: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unable to get index -- got status %q", resp.Status) + } + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to get index -- unable to read body %w", err) + } + + var index Index + if err := yaml.Unmarshal(responseBody, &index); err != nil { + return nil, fmt.Errorf("unable to unmarshal index: %w", err) + } + return &index, nil +} diff --git a/tools/setup-envtest/remote/read_body.go b/tools/setup-envtest/remote/read_body.go new file mode 100644 index 0000000000..1c71102897 --- /dev/null +++ b/tools/setup-envtest/remote/read_body.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2024 The Kubernetes Authors + +package remote + +import ( + "crypto/md5" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + + "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" +) + +func readBody(resp *http.Response, out io.Writer, archiveName string, platform versions.PlatformItem) error { + if platform.Hash != nil { + // stream in chunks to do the checksum, don't load the whole thing into + // memory to avoid causing issues with big files. + buf := make([]byte, 32*1024) // 32KiB, same as io.Copy + var hasher hash.Hash + switch platform.Hash.Type { + case versions.SHA512HashType: + hasher = sha512.New() + case versions.MD5HashType: + hasher = md5.New() + default: + return fmt.Errorf("hash type %s not implemented", platform.Hash.Type) + } + for cont := true; cont; { + amt, err := resp.Body.Read(buf) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("unable read next chunk of %s: %w", archiveName, err) + } + if amt > 0 { + // checksum never returns errors according to docs + hasher.Write(buf[:amt]) + if _, err := out.Write(buf[:amt]); err != nil { + return fmt.Errorf("unable write next chunk of %s: %w", archiveName, err) + } + } + cont = amt > 0 && !errors.Is(err, io.EOF) + } + + var sum string + switch platform.Hash.Encoding { + case versions.Base64HashEncoding: + sum = base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + case versions.HexHashEncoding: + sum = hex.EncodeToString(hasher.Sum(nil)) + default: + return fmt.Errorf("hash encoding %s not implemented", platform.Hash.Encoding) + } + if sum != platform.Hash.Value { + return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (reported)", archiveName, sum, platform.Hash.Value) + } + } else if _, err := io.Copy(out, resp.Body); err != nil { + return fmt.Errorf("unable to download %s: %w", archiveName, err) + } + return nil +} diff --git a/tools/setup-envtest/store/store.go b/tools/setup-envtest/store/store.go index e6f258e4ac..bb5a1f7bcd 100644 --- a/tools/setup-envtest/store/store.go +++ b/tools/setup-envtest/store/store.go @@ -38,7 +38,7 @@ func (i Item) String() string { } // Filter is a version spec & platform selector (i.e. platform -// potentially with wilcards) to filter store items. +// potentially with wildcards) to filter store items. type Filter struct { Version versions.Spec Platform versions.Platform @@ -174,7 +174,7 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr } if err := func() error { // IIFE to get the defer properly in a loop defer binOut.Close() - if _, err := io.Copy(binOut, tarReader); err != nil { //nolint:gosec + if _, err := io.Copy(binOut, tarReader); err != nil { return fmt.Errorf("unable to write file %s from archive to disk for version-platform pair %s", targetPath, itemName) } return nil @@ -182,7 +182,7 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr return err } } - if err != nil && !errors.Is(err, io.EOF) { + if err != nil && !errors.Is(err, io.EOF) { //nolint:govet return fmt.Errorf("unable to finish un-tar-ing the downloaded archive: %w", err) } log.V(1).Info("unpacked archive") diff --git a/tools/setup-envtest/store/store_suite_test.go b/tools/setup-envtest/store/store_suite_test.go index c2795a3227..649c22d545 100644 --- a/tools/setup-envtest/store/store_suite_test.go +++ b/tools/setup-envtest/store/store_suite_test.go @@ -40,8 +40,8 @@ func zapLogger() logr.Logger { return zapr.NewLogger(zapLog) } -func logCtx() context.Context { - return logr.NewContext(context.Background(), testLog) +func logCtx(ctx context.Context) context.Context { + return logr.NewContext(ctx, testLog) } func TestStore(t *testing.T) { diff --git a/tools/setup-envtest/store/store_test.go b/tools/setup-envtest/store/store_test.go index d5607aede6..575d49dd3b 100644 --- a/tools/setup-envtest/store/store_test.go +++ b/tools/setup-envtest/store/store_test.go @@ -47,23 +47,23 @@ var _ = Describe("Store", func() { } }) Describe("initialization", func() { - It("should ensure the repo root exists", func() { + It("should ensure the repo root exists", func(ctx SpecContext) { // remove the old dir Expect(st.Root.RemoveAll("")).To(Succeed(), "should be able to remove the store before trying to initialize") - Expect(st.Initialize(logCtx())).To(Succeed(), "initialization should succeed") + Expect(st.Initialize(logCtx(ctx))).To(Succeed(), "initialization should succeed") Expect(st.Root.Stat("k8s")).NotTo(BeNil(), "store's binary dir should exist") }) - It("should be fine if the repo root already exists", func() { - Expect(st.Initialize(logCtx())).To(Succeed()) + It("should be fine if the repo root already exists", func(ctx SpecContext) { + Expect(st.Initialize(logCtx(ctx))).To(Succeed()) }) }) Describe("listing items", func() { - It("should filter results by the given filter, sorted in version order (newest first)", func() { + It("should filter results by the given filter, sorted in version order (newest first)", func(ctx SpecContext) { sel, err := versions.FromExpr("<=1.16") Expect(err).NotTo(HaveOccurred(), "should be able to construct <=1.16 selector") - Expect(st.List(logCtx(), store.Filter{ + Expect(st.List(logCtx(ctx), store.Filter{ Version: sel, Platform: versions.Platform{OS: "*", Arch: "amd64"}, })).To(Equal([]store.Item{ @@ -73,16 +73,16 @@ var _ = Describe("Store", func() { {Version: ver(1, 14, 26), Platform: versions.Platform{OS: "linux", Arch: "amd64"}}, })) }) - It("should skip non-folders in the store", func() { + It("should skip non-folders in the store", func(ctx SpecContext) { Expect(afero.WriteFile(st.Root, "k8s/2.3.6-linux-amd128", []byte{0x01}, fs.ModePerm)).To(Succeed(), "should be able to create a non-store file in the store directory") - Expect(st.List(logCtx(), store.Filter{ + Expect(st.List(logCtx(ctx), store.Filter{ Version: versions.AnyVersion, Platform: versions.Platform{OS: "linux", Arch: "amd128"}, })).To(BeEmpty()) }) - It("should skip non-matching names in the store", func() { + It("should skip non-matching names in the store", func(ctx SpecContext) { Expect(st.Root.Mkdir("k8s/somedir-2.3.6-linux-amd128", fs.ModePerm)).To(Succeed(), "should be able to create a non-store file in the store directory") - Expect(st.List(logCtx(), store.Filter{ + Expect(st.List(logCtx(ctx), store.Filter{ Version: versions.AnyVersion, Platform: versions.Platform{OS: "linux", Arch: "amd128"}, })).To(BeEmpty()) }) @@ -90,10 +90,10 @@ var _ = Describe("Store", func() { Describe("removing items", func() { var res []store.Item - BeforeEach(func() { + BeforeEach(func(ctx SpecContext) { sel, err := versions.FromExpr("<=1.16") Expect(err).NotTo(HaveOccurred(), "should be able to construct <=1.16 selector") - res, err = st.Remove(logCtx(), store.Filter{ + res, err = st.Remove(logCtx(ctx), store.Filter{ Version: sel, Platform: versions.Platform{OS: "*", Arch: "amd64"}, }) @@ -125,28 +125,30 @@ var _ = Describe("Store", func() { }) }) - Describe("adding items", func() { - It("should support .tar.gz input", func() { - Expect(st.Add(logCtx(), newItem, makeFakeArchive(newName))).To(Succeed()) + Describe("adding items (controller-tools archives)", func() { + archiveName := "envtest-v1.16.3-linux-amd64.tar.gz" + + It("should support .tar.gz input", func(ctx SpecContext) { + Expect(st.Add(logCtx(ctx), newItem, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) Expect(st.Has(newItem)).To(BeTrue(), "should have the item after adding it") }) - It("should extract binaries from the given archive to a directly to the item's directory, regardless of path", func() { - Expect(st.Add(logCtx(), newItem, makeFakeArchive(newName))).To(Succeed()) + It("should extract binaries from the given archive to a directly to the item's directory, regardless of path", func(ctx SpecContext) { + Expect(st.Add(logCtx(ctx), newItem, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) dirName := newItem.Platform.BaseName(newItem.Version) - Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "some-file"))).To(HavePrefix(newName + "some-file")) - Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "other-file"))).To(HavePrefix(newName + "other-file")) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "some-file"))).To(HavePrefix(archiveName + "some-file")) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "other-file"))).To(HavePrefix(archiveName + "other-file")) }) - It("should clean up any existing item directory before creating the new one", func() { + It("should clean up any existing item directory before creating the new one", func(ctx SpecContext) { item := localVersions[0] - Expect(st.Add(logCtx(), item, makeFakeArchive(newName))).To(Succeed()) + Expect(st.Add(logCtx(ctx), item, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) Expect(st.Root.Stat(filepath.Join("k8s", item.Platform.BaseName(item.Version)))).NotTo(BeNil(), "new files should exist") }) - It("should clean up if it errors before finishing", func() { + It("should clean up if it errors before finishing", func(ctx SpecContext) { item := localVersions[0] - Expect(st.Add(logCtx(), item, new(bytes.Buffer))).NotTo(Succeed(), "should fail to extract") + Expect(st.Add(logCtx(ctx), item, new(bytes.Buffer))).NotTo(Succeed(), "should fail to extract") _, err := st.Root.Stat(filepath.Join("k8s", item.Platform.BaseName(item.Version))) Expect(err).To(HaveOccurred(), "the binaries dir for the item should be gone") @@ -187,8 +189,6 @@ var ( Version: ver(1, 16, 3), Platform: versions.Platform{OS: "linux", Arch: "amd64"}, } - - newName = "kubebuilder-tools-1.16.3-linux-amd64.tar.gz" ) func ver(major, minor, patch int) versions.Concrete { @@ -199,13 +199,13 @@ func ver(major, minor, patch int) versions.Concrete { } } -func makeFakeArchive(magic string) io.Reader { +func makeFakeArchive(magic, relativePath string) io.Reader { out := new(bytes.Buffer) gzipWriter := gzip.NewWriter(out) tarWriter := tar.NewWriter(gzipWriter) Expect(tarWriter.WriteHeader(&tar.Header{ Typeflag: tar.TypeDir, - Name: "kubebuilder/bin/", // so we can ensure we skip non-files + Name: relativePath, // so we can ensure we skip non-files Mode: 0777, })).To(Succeed()) for _, fileName := range []string{"some-file", "other-file"} { @@ -218,9 +218,9 @@ func makeFakeArchive(magic string) io.Reader { panic(err) } - // write to kubebuilder/bin/fileName + // write to relativePath/fileName err := tarWriter.WriteHeader(&tar.Header{ - Name: "kubebuilder/bin/" + fileName, + Name: relativePath + fileName, Size: int64(len(chunk[:])), Mode: 0777, // so we can check that we fix this later }) diff --git a/tools/setup-envtest/version/version.go b/tools/setup-envtest/version/version.go new file mode 100644 index 0000000000..1d148b085d --- /dev/null +++ b/tools/setup-envtest/version/version.go @@ -0,0 +1,21 @@ +package version + +import "runtime/debug" + +// Version to be set using ldflags: +// -ldflags "-X sigs.k8s.io/controller-runtime/tools/setup-envtest/version.version=v1.0.0" +// falls back to module information is unse +var version = "" + +// Version returns the version of the main module +func Version() string { + if version != "" { + return version + } + info, ok := debug.ReadBuildInfo() + if !ok || info == nil || info.Main.Version == "" { + // binary has not been built with module support or doesn't contain a version. + return "(unknown)" + } + return info.Main.Version +} diff --git a/pkg/config/config_suite_test.go b/tools/setup-envtest/version/version_suite_test.go similarity index 83% rename from pkg/config/config_suite_test.go rename to tools/setup-envtest/version/version_suite_test.go index 8df933ba9d..99c623e8d4 100644 --- a/pkg/config/config_suite_test.go +++ b/tools/setup-envtest/version/version_suite_test.go @@ -1,9 +1,10 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -11,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package config_test +package version import ( "testing" @@ -20,7 +21,7 @@ import ( . "github.com/onsi/gomega" ) -func TestScheme(t *testing.T) { +func TestVersioning(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "Config Suite") + RunSpecs(t, "Test Version Suite") } diff --git a/tools/setup-envtest/version/version_test.go b/tools/setup-envtest/version/version_test.go new file mode 100644 index 0000000000..4178cac870 --- /dev/null +++ b/tools/setup-envtest/version/version_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and + +limitations under the License. +*/ + +package version + +import ( + "runtime/debug" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("TestVersion", func() { + + info, ok := debug.ReadBuildInfo() + Expect(ok).To(BeTrue()) + tests := map[string]struct { + version string + expected string + }{ + "empty returns build info": { + version: "", + expected: info.Main.Version, + }, + "set to a value returns it": { + version: "1.2.3", + expected: "1.2.3", + }, + } + for name, tc := range tests { + It("Version set to "+name, func() { + versionBackup := version + defer func() { + version = versionBackup + }() + version = tc.version + result := Version() + Expect(result).To(Equal(tc.expected)) + }) + } +}) diff --git a/tools/setup-envtest/versions/misc_test.go b/tools/setup-envtest/versions/misc_test.go index 8a97de0410..a609f4dc60 100644 --- a/tools/setup-envtest/versions/misc_test.go +++ b/tools/setup-envtest/versions/misc_test.go @@ -95,7 +95,7 @@ var _ = Describe("Platform", func() { Specify("knows how to produce an archive name", func() { plat := Platform{OS: "linux", Arch: "amd64"} ver := Concrete{Major: 1, Minor: 16, Patch: 3} - Expect(plat.ArchiveName(ver)).To(Equal("kubebuilder-tools-1.16.3-linux-amd64.tar.gz")) + Expect(plat.ArchiveName(ver)).To(Equal("envtest-v1.16.3-linux-amd64.tar.gz")) }) Describe("parsing", func() { @@ -110,14 +110,14 @@ var _ = Describe("Platform", func() { Expect(ver).To(BeNil()) }) }) - Context("for archive names", func() { - It("should accept strings of the form kubebuilder-tools-x.y.z-os-arch.tar.gz", func() { - ver, plat := ExtractWithPlatform(ArchiveRE, "kubebuilder-tools-1.16.3-linux-amd64.tar.gz") + Context("for archive names (controller-tools)", func() { + It("should accept strings of the form envtest-vx.y.z-os-arch.tar.gz", func() { + ver, plat := ExtractWithPlatform(ArchiveRE, "envtest-v1.16.3-linux-amd64.tar.gz") Expect(ver).To(Equal(&Concrete{Major: 1, Minor: 16, Patch: 3})) Expect(plat).To(Equal(Platform{OS: "linux", Arch: "amd64"})) }) It("should reject nonsense strings", func() { - ver, _ := ExtractWithPlatform(ArchiveRE, "kubebuilder-tools-1.16.3-linux-amd64.tar.sum") + ver, _ := ExtractWithPlatform(ArchiveRE, "envtest-v1.16.3-linux-amd64.tar.sum") Expect(ver).To(BeNil()) }) }) diff --git a/tools/setup-envtest/versions/parse.go b/tools/setup-envtest/versions/parse.go index c053bf8757..cd25710b2b 100644 --- a/tools/setup-envtest/versions/parse.go +++ b/tools/setup-envtest/versions/parse.go @@ -17,8 +17,6 @@ var ( // ConcreteVersionRE matches a concrete version anywhere in the string. ConcreteVersionRE = regexp.MustCompile(`(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)`) - // OnlyConcreteVersionRE matches a string that's just a concrete version. - OnlyConcreteVersionRE = regexp.MustCompile(`^` + ConcreteVersionRE.String() + `$`) ) // FromExpr extracts a version from a string in the form of a semver version, @@ -109,7 +107,7 @@ func PatchSelectorFromMatch(match []string, re *regexp.Regexp) PatchSelector { panic("invalid input passed as patch selector (invalid state)") } - // patch is optional, means wilcard if left off + // patch is optional, means wildcard if left off patch := AnyPoint if patchRaw := match[re.SubexpIndex("patch")]; patchRaw != "" { patch = PointVersionFromValidString(patchRaw) diff --git a/tools/setup-envtest/versions/platform.go b/tools/setup-envtest/versions/platform.go index 16c08b38ff..1cfbd05c06 100644 --- a/tools/setup-envtest/versions/platform.go +++ b/tools/setup-envtest/versions/platform.go @@ -38,16 +38,53 @@ func (p Platform) BaseName(ver Concrete) string { // ArchiveName returns the full archive name for this version and platform. func (p Platform) ArchiveName(ver Concrete) string { - return "kubebuilder-tools-" + p.BaseName(ver) + ".tar.gz" + return "envtest-v" + p.BaseName(ver) + ".tar.gz" } // PlatformItem represents a platform with corresponding // known metadata for its download. type PlatformItem struct { Platform - MD5 string + + *Hash } +// Hash of an archive with envtest binaries. +type Hash struct { + // Type of the hash. + // controller-tools uses SHA512HashType. + Type HashType + + // Encoding of the hash value. + // controller-tools uses HexHashEncoding. + Encoding HashEncoding + + // Value of the hash. + Value string +} + +// HashType is the type of a hash. +type HashType string + +const ( + // SHA512HashType represents a sha512 hash + SHA512HashType HashType = "sha512" + + // MD5HashType represents a md5 hash + MD5HashType HashType = "md5" +) + +// HashEncoding is the encoding of a hash +type HashEncoding string + +const ( + // Base64HashEncoding represents base64 encoding + Base64HashEncoding HashEncoding = "base64" + + // HexHashEncoding represents hex encoding + HexHashEncoding HashEncoding = "hex" +) + // Set is a concrete version and all the corresponding platforms that it's available for. type Set struct { Version Concrete @@ -81,5 +118,6 @@ var ( // VersionPlatformRE matches concrete version-platform strings. VersionPlatformRE = regexp.MustCompile(`^` + versionPlatformREBase + `$`) // ArchiveRE matches concrete version-platform.tar.gz strings. - ArchiveRE = regexp.MustCompile(`^kubebuilder-tools-` + versionPlatformREBase + `\.tar\.gz$`) + // The archives published to GitHub releases by controller-tools use the "envtest-v" prefix (e.g. "envtest-v1.30.0-darwin-amd64.tar.gz"). + ArchiveRE = regexp.MustCompile(`^envtest-v` + versionPlatformREBase + `\.tar\.gz$`) ) diff --git a/tools/setup-envtest/versions/version.go b/tools/setup-envtest/versions/version.go index 582ed7794e..945a95006f 100644 --- a/tools/setup-envtest/versions/version.go +++ b/tools/setup-envtest/versions/version.go @@ -72,7 +72,7 @@ func (s PatchSelector) AsConcrete() *Concrete { return &Concrete{ Major: s.Major, Minor: s.Minor, - Patch: int(s.Patch), // safe to cast, we've just checked wilcards above + Patch: int(s.Patch), // safe to cast, we've just checked wildcards above } } diff --git a/tools/setup-envtest/workflows/workflows.go b/tools/setup-envtest/workflows/workflows.go index fdabd995ae..fb9123d269 100644 --- a/tools/setup-envtest/workflows/workflows.go +++ b/tools/setup-envtest/workflows/workflows.go @@ -5,11 +5,13 @@ package workflows import ( "context" + "fmt" "io" "github.com/go-logr/logr" envp "sigs.k8s.io/controller-runtime/tools/setup-envtest/env" + "sigs.k8s.io/controller-runtime/tools/setup-envtest/version" ) // Use is a workflow that prints out information about stored @@ -85,3 +87,12 @@ func (f Sideload) Do(env *envp.Env) { env.Sideload(ctx, f.Input) env.PrintInfo(f.PrintFormat) } + +// Version is the workflow that shows the current binary version +// of setup-envtest. +type Version struct{} + +// Do executes the workflow. +func (v Version) Do(env *envp.Env) { + fmt.Fprintf(env.Out, "setup-envtest version: %s\n", version.Version()) +} diff --git a/tools/setup-envtest/workflows/workflows_test.go b/tools/setup-envtest/workflows/workflows_test.go index a0bd7321f7..435ae24285 100644 --- a/tools/setup-envtest/workflows/workflows_test.go +++ b/tools/setup-envtest/workflows/workflows_test.go @@ -5,15 +5,18 @@ package workflows_test import ( "bytes" + "fmt" "io/fs" "path/filepath" + "runtime/debug" + "sort" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" "github.com/spf13/afero" - + "k8s.io/apimachinery/pkg/util/sets" envp "sigs.k8s.io/controller-runtime/tools/setup-envtest/env" "sigs.k8s.io/controller-runtime/tools/setup-envtest/remote" "sigs.k8s.io/controller-runtime/tools/setup-envtest/store" @@ -48,14 +51,22 @@ const ( var _ = Describe("Workflows", func() { var ( - env *envp.Env - out *bytes.Buffer - server *ghttp.Server - remoteItems []item + env *envp.Env + out *bytes.Buffer + server *ghttp.Server + remoteHTTPItems itemsHTTP ) BeforeEach(func() { out = new(bytes.Buffer) baseFs := afero.Afero{Fs: afero.NewMemMapFs()} + + server = ghttp.NewServer() + + client := &remote.HTTPClient{ + Log: testLog.WithName("http-client"), + IndexURL: fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml"), + } + env = &envp.Env{ Log: testLog, VerifySum: true, // on by default @@ -68,21 +79,14 @@ var _ = Describe("Workflows", func() { Arch: "amd64", }, }, - Client: &remote.Client{ - Log: testLog.WithName("remote-client"), - Bucket: "kubebuilder-tools-test", // test custom bucket functionality too - Server: "localhost:-1", - Insecure: true, // no https in httptest :-( - }, + Client: client, } - server = ghttp.NewServer() - env.Client.Server = server.Addr() fakeStore(env.FS, testStorePath) - remoteItems = remoteVersions + remoteHTTPItems = remoteVersionsHTTP }) JustBeforeEach(func() { - handleRemoteVersions(server, remoteItems) + handleRemoteVersionsHTTP(server, remoteHTTPItems) }) AfterEach(func() { server.Close() @@ -245,18 +249,22 @@ var _ = Describe("Workflows", func() { Describe("verifying the checksum", func() { BeforeEach(func() { - remoteItems = append(remoteItems, item{ - meta: bucketObject{ - Name: "kubebuilder-tools-86.75.309-linux-amd64.tar.gz", - Hash: "nottherightone!", + // Recreate remoteHTTPItems to not impact others tests. + remoteHTTPItems = makeContentsHTTP(remoteNamesHTTP) + remoteHTTPItems.index.Releases["v86.75.309"] = map[string]remote.Archive{ + "envtest-v86.75.309-linux-amd64.tar.gz": { + SelfLink: "not used in this test", + Hash: "nottherightone!", }, - contents: remoteItems[0].contents, // need a valid tar.gz file to not error from that - }) + } + // need a valid tar.gz file to not error from that + remoteHTTPItems.contents["envtest-v86.75.309-linux-amd64.tar.gz"] = remoteHTTPItems.contents["envtest-v1.10-darwin-amd64.tar.gz"] + env.Version = versions.Spec{ Selector: ver(86, 75, 309), } }) - Specify("when enabled, should fail if the downloaded md5 checksum doesn't match", func() { + Specify("when enabled, should fail if the downloaded hash doesn't match", func() { defer shouldHaveError() flow.Do(env) }) @@ -310,14 +318,31 @@ var _ = Describe("Workflows", func() { {"(installed)", "v1.16.1", "linux/amd64"}, {"(installed)", "v1.16.0", "linux/amd64"}, })) - }) }) Context("when downloads are enabled", func() { Context("when sorting", func() { BeforeEach(func() { - // shorten the list a bit for expediency - remoteItems = remoteItems[:7] + // Recreate remoteHTTPItems to not impact others tests. + remoteHTTPItems = makeContentsHTTP(remoteNamesHTTP) + // Also only keep the first 7 items. + // Get the first 7 archive names + var archiveNames []string + for _, release := range remoteHTTPItems.index.Releases { + for archiveName := range release { + archiveNames = append(archiveNames, archiveName) + } + } + sort.Strings(archiveNames) + archiveNamesSet := sets.Set[string]{}.Insert(archiveNames[:7]...) + // Delete all other archives + for _, release := range remoteHTTPItems.index.Releases { + for archiveName := range release { + if !archiveNamesSet.Has(archiveName) { + delete(release, archiveName) + } + } + } }) It("should sort local & remote by version", func() { env.Version = versions.AnyVersion @@ -338,7 +363,6 @@ var _ = Describe("Workflows", func() { {"(available)", "v1.10.1", "darwin/amd64"}, {"(available)", "v1.10.1", "linux/amd64"}, })) - }) }) It("should skip non-matching remote contents", func() { @@ -354,7 +378,6 @@ var _ = Describe("Workflows", func() { {"(installed)", "v1.16.0", "linux/amd64"}, {"(available)", "v1.16.4", "linux/amd64"}, })) - }) }) }) @@ -381,13 +404,17 @@ var _ = Describe("Workflows", func() { Describe("sideload", func() { var ( flow wf.Sideload - // remote version fake contents are prefixed by the - // name for easier debugging, so we can use that here - expectedPrefix = remoteVersions[0].meta.Name ) + + // hard coding to one of the archives in remoteVersionsHTTP as we can't pick the "first" of a map. + expectedPrefix := "envtest-v1.10-darwin-amd64.tar.gz" + BeforeEach(func() { server.Close() // ensure no network - flow.Input = bytes.NewReader(remoteVersions[0].contents) + + content := remoteVersionsHTTP.contents[expectedPrefix] + + flow.Input = bytes.NewReader(content) flow.PrintFormat = envp.PrintPath }) It("should initialize the store if it doesn't exist", func() { @@ -417,4 +444,16 @@ var _ = Describe("Workflows", func() { Expect(string(outContents)).To(HavePrefix(expectedPrefix), "should have the debugging prefix") }) }) + + Describe("version", func() { + It("should print out the version if the RELEASE_TAG is empty", func() { + v := wf.Version{} + v.Do(env) + info, ok := debug.ReadBuildInfo() + Expect(ok).To(BeTrue()) + Expect(out.String()).ToNot(BeEmpty()) + Expect(out.String()).To(Equal(fmt.Sprintf("setup-envtest version: %s\n", info.Main.Version))) + }) + }) + }) diff --git a/tools/setup-envtest/workflows/workflows_testutils_test.go b/tools/setup-envtest/workflows/workflows_testutils_test.go index 93e832d763..6bf6db38c3 100644 --- a/tools/setup-envtest/workflows/workflows_testutils_test.go +++ b/tools/setup-envtest/workflows/workflows_testutils_test.go @@ -7,9 +7,10 @@ import ( "archive/tar" "bytes" "compress/gzip" - "crypto/md5" //nolint:gosec "crypto/rand" - "encoding/base64" + "crypto/sha512" + "encoding/hex" + "fmt" "net/http" "path/filepath" @@ -17,50 +18,73 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" "github.com/spf13/afero" + "sigs.k8s.io/controller-runtime/tools/setup-envtest/remote" + "sigs.k8s.io/yaml" "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" ) var ( - remoteNames = []string{ - "kubebuilder-tools-1.10-darwin-amd64.tar.gz", - "kubebuilder-tools-1.10-linux-amd64.tar.gz", - "kubebuilder-tools-1.10.1-darwin-amd64.tar.gz", - "kubebuilder-tools-1.10.1-linux-amd64.tar.gz", - "kubebuilder-tools-1.11.0-darwin-amd64.tar.gz", - "kubebuilder-tools-1.11.0-linux-amd64.tar.gz", - "kubebuilder-tools-1.11.1-potato-cherrypie.tar.gz", - "kubebuilder-tools-1.12.3-darwin-amd64.tar.gz", - "kubebuilder-tools-1.12.3-linux-amd64.tar.gz", - "kubebuilder-tools-1.13.1-darwin-amd64.tar.gz", - "kubebuilder-tools-1.13.1-linux-amd64.tar.gz", - "kubebuilder-tools-1.14.1-darwin-amd64.tar.gz", - "kubebuilder-tools-1.14.1-linux-amd64.tar.gz", - "kubebuilder-tools-1.15.5-darwin-amd64.tar.gz", - "kubebuilder-tools-1.15.5-linux-amd64.tar.gz", - "kubebuilder-tools-1.16.4-darwin-amd64.tar.gz", - "kubebuilder-tools-1.16.4-linux-amd64.tar.gz", - "kubebuilder-tools-1.17.9-darwin-amd64.tar.gz", - "kubebuilder-tools-1.17.9-linux-amd64.tar.gz", - "kubebuilder-tools-1.19.0-darwin-amd64.tar.gz", - "kubebuilder-tools-1.19.0-linux-amd64.tar.gz", - "kubebuilder-tools-1.19.2-darwin-amd64.tar.gz", - "kubebuilder-tools-1.19.2-linux-amd64.tar.gz", - "kubebuilder-tools-1.19.2-linux-arm64.tar.gz", - "kubebuilder-tools-1.19.2-linux-ppc64le.tar.gz", - "kubebuilder-tools-1.20.2-darwin-amd64.tar.gz", - "kubebuilder-tools-1.20.2-linux-amd64.tar.gz", - "kubebuilder-tools-1.20.2-linux-arm64.tar.gz", - "kubebuilder-tools-1.20.2-linux-ppc64le.tar.gz", - "kubebuilder-tools-1.9-darwin-amd64.tar.gz", - "kubebuilder-tools-1.9-linux-amd64.tar.gz", - "kubebuilder-tools-v1.19.2-darwin-amd64.tar.gz", - "kubebuilder-tools-v1.19.2-linux-amd64.tar.gz", - "kubebuilder-tools-v1.19.2-linux-arm64.tar.gz", - "kubebuilder-tools-v1.19.2-linux-ppc64le.tar.gz", + remoteNamesHTTP = remote.Index{ + Releases: map[string]remote.Release{ + "v1.10.0": map[string]remote.Archive{ + "envtest-v1.10-darwin-amd64.tar.gz": {}, + "envtest-v1.10-linux-amd64.tar.gz": {}, + }, + "v1.10.1": map[string]remote.Archive{ + "envtest-v1.10.1-darwin-amd64.tar.gz": {}, + "envtest-v1.10.1-linux-amd64.tar.gz": {}, + }, + "v1.11.0": map[string]remote.Archive{ + "envtest-v1.11.0-darwin-amd64.tar.gz": {}, + "envtest-v1.11.0-linux-amd64.tar.gz": {}, + }, + "v1.11.1": map[string]remote.Archive{ + "envtest-v1.11.1-potato-cherrypie.tar.gz": {}, + }, + "v1.12.3": map[string]remote.Archive{ + "envtest-v1.12.3-darwin-amd64.tar.gz": {}, + "envtest-v1.12.3-linux-amd64.tar.gz": {}, + }, + "v1.13.1": map[string]remote.Archive{ + "envtest-v1.13.1-darwin-amd64.tar.gz": {}, + "envtest-v1.13.1-linux-amd64.tar.gz": {}, + }, + "v1.14.1": map[string]remote.Archive{ + "envtest-v1.14.1-darwin-amd64.tar.gz": {}, + "envtest-v1.14.1-linux-amd64.tar.gz": {}, + }, + "v1.15.5": map[string]remote.Archive{ + "envtest-v1.15.5-darwin-amd64.tar.gz": {}, + "envtest-v1.15.5-linux-amd64.tar.gz": {}, + }, + "v1.16.4": map[string]remote.Archive{ + "envtest-v1.16.4-darwin-amd64.tar.gz": {}, + "envtest-v1.16.4-linux-amd64.tar.gz": {}, + }, + "v1.17.9": map[string]remote.Archive{ + "envtest-v1.17.9-darwin-amd64.tar.gz": {}, + "envtest-v1.17.9-linux-amd64.tar.gz": {}, + }, + "v1.19.0": map[string]remote.Archive{ + "envtest-v1.19.0-darwin-amd64.tar.gz": {}, + "envtest-v1.19.0-linux-amd64.tar.gz": {}, + }, + "v1.19.2": map[string]remote.Archive{ + "envtest-v1.19.2-darwin-amd64.tar.gz": {}, + "envtest-v1.19.2-linux-amd64.tar.gz": {}, + "envtest-v1.19.2-linux-arm64.tar.gz": {}, + "envtest-v1.19.2-linux-ppc64le.tar.gz": {}, + }, + "v1.20.2": map[string]remote.Archive{ + "envtest-v1.20.2-darwin-amd64.tar.gz": {}, + "envtest-v1.20.2-linux-amd64.tar.gz": {}, + "envtest-v1.20.2-linux-arm64.tar.gz": {}, + "envtest-v1.20.2-linux-ppc64le.tar.gz": {}, + }, + }, } - - remoteVersions = makeContents(remoteNames) + remoteVersionsHTTP = makeContentsHTTP(remoteNamesHTTP) // keep this sorted. localVersions = []versions.Set{ @@ -84,41 +108,49 @@ var ( } ) -type item struct { - meta bucketObject - contents []byte +type itemsHTTP struct { + index remote.Index + contents map[string][]byte } -// objectList is the parts we need of the GCS "list-objects-in-bucket" endpoint. -type objectList struct { - Items []bucketObject `json:"items"` -} +func makeContentsHTTP(index remote.Index) itemsHTTP { + // This creates a new copy of the index so modifying the index + // in some tests doesn't affect others. + res := itemsHTTP{ + index: remote.Index{ + Releases: map[string]remote.Release{}, + }, + contents: map[string][]byte{}, + } -// bucketObject is the parts we need of the GCS object metadata. -type bucketObject struct { - Name string `json:"name"` - Hash string `json:"md5Hash"` -} + for releaseVersion, releases := range index.Releases { + res.index.Releases[releaseVersion] = remote.Release{} + for archiveName := range releases { + var chunk [1024 * 48]byte // 1.5 times our chunk read size in GetVersion + copy(chunk[:], archiveName) + if _, err := rand.Read(chunk[len(archiveName):]); err != nil { + panic(err) + } + content, hash := verWithHTTP(chunk[:]) -func makeContents(names []string) []item { - res := make([]item, len(names)) - for i, name := range names { - var chunk [1024 * 48]byte // 1.5 times our chunk read size in GetVersion - copy(chunk[:], name) - if _, err := rand.Read(chunk[len(name):]); err != nil { - panic(err) + res.index.Releases[releaseVersion][archiveName] = remote.Archive{ + Hash: hash, + // Note: Only storing the name of the archive for now. + // This will be expanded later to a full URL once the server is running. + SelfLink: archiveName, + } + res.contents[archiveName] = content } - res[i] = verWith(name, chunk[:]) } return res } -func verWith(name string, contents []byte) item { +func verWithHTTP(contents []byte) ([]byte, string) { out := new(bytes.Buffer) gzipWriter := gzip.NewWriter(out) tarWriter := tar.NewWriter(gzipWriter) err := tarWriter.WriteHeader(&tar.Header{ - Name: "kubebuilder/bin/some-file", + Name: "controller-tools/envtest/some-file", Size: int64(len(contents)), Mode: 0777, // so we can check that we fix this later }) @@ -131,35 +163,49 @@ func verWith(name string, contents []byte) item { } tarWriter.Close() gzipWriter.Close() - res := item{ - meta: bucketObject{Name: name}, - contents: out.Bytes(), - } - hash := md5.Sum(res.contents) //nolint:gosec - res.meta.Hash = base64.StdEncoding.EncodeToString(hash[:]) - return res + content := out.Bytes() + // controller-tools is using sha512 + hash := sha512.Sum512(content) + hashEncoded := hex.EncodeToString(hash[:]) + return content, hashEncoded } -func handleRemoteVersions(server *ghttp.Server, versions []item) { - list := objectList{Items: make([]bucketObject, len(versions))} - for i, ver := range versions { - ver := ver // copy to avoid capturing the iteration variable - list.Items[i] = ver.meta - server.RouteToHandler("GET", "/storage/v1/b/kubebuilder-tools-test/o/"+ver.meta.Name, func(resp http.ResponseWriter, req *http.Request) { - if req.URL.Query().Get("alt") == "media" { - resp.WriteHeader(http.StatusOK) - Expect(resp.Write(ver.contents)).To(Equal(len(ver.contents))) - } else { - ghttp.RespondWithJSONEncoded( - http.StatusOK, - ver.meta, - )(resp, req) +func handleRemoteVersionsHTTP(server *ghttp.Server, items itemsHTTP) { + if server.HTTPTestServer == nil { + // Just return for test cases where server is closed in BeforeEach. Otherwise server.Addr() below panics. + return + } + + // The index from items contains only relative SelfLinks. + // finalIndex will contain the full links based on server.Addr(). + finalIndex := remote.Index{ + Releases: map[string]remote.Release{}, + } + + for releaseVersion, releases := range items.index.Releases { + finalIndex.Releases[releaseVersion] = remote.Release{} + + for archiveName, archive := range releases { + finalIndex.Releases[releaseVersion][archiveName] = remote.Archive{ + Hash: archive.Hash, + SelfLink: fmt.Sprintf("http://%s/%s", server.Addr(), archive.SelfLink), } - }) + content := items.contents[archiveName] + + // Note: Using the relative path from archive here instead of the full path. + server.RouteToHandler("GET", "/"+archive.SelfLink, func(resp http.ResponseWriter, req *http.Request) { + resp.WriteHeader(http.StatusOK) + Expect(resp.Write(content)).To(Equal(len(content))) + }) + } } - server.RouteToHandler("GET", "/storage/v1/b/kubebuilder-tools-test/o", ghttp.RespondWithJSONEncoded( + + indexYAML, err := yaml.Marshal(finalIndex) + Expect(err).ToNot(HaveOccurred()) + + server.RouteToHandler("GET", "/envtest-releases.yaml", ghttp.RespondWith( http.StatusOK, - list, + indexYAML, )) }