diff --git a/.features/pending/disable-write-back-informer.md b/.features/pending/disable-write-back-informer.md new file mode 100644 index 000000000000..1627e73edd4a --- /dev/null +++ b/.features/pending/disable-write-back-informer.md @@ -0,0 +1,7 @@ +Description: Disable write back informer by default +Author: [Eduardo Rodrigues](https://github.com/eduardodbr) +Component: General +Issues: 12352 + +Update the controller’s default behavior to disable the write-back informer. We’ve seen several cases of unexpected behavior that appear to be caused by the write-back mechanism, and Kubernetes docs recommend avoiding writes to the informer store. Although turning it off may increase the frequency of 409 Conflict errors, it should help reduce unpredictable controller behavior. + diff --git a/docs/environment-variables.md b/docs/environment-variables.md index a3a75e8d5f04..c9b020bd2fb6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -32,7 +32,7 @@ This document outlines environment variables that can be used to customize behav | `EXPRESSION_TEMPLATES` | `bool` | `true` | Escape hatch to disable expression templates. | | `EVENT_AGGREGATION_WITH_ANNOTATIONS` | `bool` | `false` | Whether event annotations will be used when aggregating events. | | `GZIP_IMPLEMENTATION` | `string` | `PGZip` | The implementation of compression/decompression. Currently only "`PGZip`" and "`GZip`" are supported. | -| `INFORMER_WRITE_BACK` | `bool` | `true` | Whether to write back to informer instead of catching up. | +| `INFORMER_WRITE_BACK` | `bool` | `false` | Whether to write back to informer instead of catching up. | | `HEALTHZ_AGE` | `time.Duration` | `5m` | How old a un-reconciled workflow is to report unhealthy. | | `INDEX_WORKFLOW_SEMAPHORE_KEYS` | `bool` | `true` | Whether or not to index semaphores. | | `LEADER_ELECTION_IDENTITY` | `string` | Controller's `metadata.name` | The ID used for workflow controllers to elect a leader. | diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go index 07b0632eddf0..f66f8a7e4630 100644 --- a/workflow/controller/operator.go +++ b/workflow/controller/operator.go @@ -798,14 +798,15 @@ func (woc *wfOperationCtx) persistUpdates(ctx context.Context) { woc.log.WithFields(logging.Fields{"resourceVersion": woc.wf.ResourceVersion, "phase": woc.wf.Status.Phase}).Info(ctx, "Workflow update successful") switch os.Getenv("INFORMER_WRITE_BACK") { - // By default we write back (as per v2.11), this does not reduce errors, but does reduce + // this does not reduce errors, but does reduce // conflicts and therefore we log fewer warning messages. - case "", "true": + case "true": if err := woc.writeBackToInformer(); err != nil { woc.markWorkflowError(ctx, err) return } - case "false": + // no longer write back to informer cache as default (as per v4.0) + case "", "false": time.Sleep(1 * time.Second) }