From 7a6f68176dc075f269727079e4a194533cf7d2ff Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Wed, 26 Nov 2025 00:18:37 +0000 Subject: [PATCH 1/3] feat: disable write back informer by default Signed-off-by: Eduardo Rodrigues --- docs/environment-variables.md | 2 +- workflow/controller/operator.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index a3a75e8d5f04..c9b020bd2fb6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -32,7 +32,7 @@ This document outlines environment variables that can be used to customize behav | `EXPRESSION_TEMPLATES` | `bool` | `true` | Escape hatch to disable expression templates. | | `EVENT_AGGREGATION_WITH_ANNOTATIONS` | `bool` | `false` | Whether event annotations will be used when aggregating events. | | `GZIP_IMPLEMENTATION` | `string` | `PGZip` | The implementation of compression/decompression. Currently only "`PGZip`" and "`GZip`" are supported. | -| `INFORMER_WRITE_BACK` | `bool` | `true` | Whether to write back to informer instead of catching up. | +| `INFORMER_WRITE_BACK` | `bool` | `false` | Whether to write back to informer instead of catching up. | | `HEALTHZ_AGE` | `time.Duration` | `5m` | How old a un-reconciled workflow is to report unhealthy. | | `INDEX_WORKFLOW_SEMAPHORE_KEYS` | `bool` | `true` | Whether or not to index semaphores. | | `LEADER_ELECTION_IDENTITY` | `string` | Controller's `metadata.name` | The ID used for workflow controllers to elect a leader. | diff --git a/workflow/controller/operator.go b/workflow/controller/operator.go index 07b0632eddf0..f66f8a7e4630 100644 --- a/workflow/controller/operator.go +++ b/workflow/controller/operator.go @@ -798,14 +798,15 @@ func (woc *wfOperationCtx) persistUpdates(ctx context.Context) { woc.log.WithFields(logging.Fields{"resourceVersion": woc.wf.ResourceVersion, "phase": woc.wf.Status.Phase}).Info(ctx, "Workflow update successful") switch os.Getenv("INFORMER_WRITE_BACK") { - // By default we write back (as per v2.11), this does not reduce errors, but does reduce + // this does not reduce errors, but does reduce // conflicts and therefore we log fewer warning messages. - case "", "true": + case "true": if err := woc.writeBackToInformer(); err != nil { woc.markWorkflowError(ctx, err) return } - case "false": + // no longer write back to informer cache as default (as per v4.0) + case "", "false": time.Sleep(1 * time.Second) } From ccc9484e67660524d96b835edcdd8b795da13204 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Wed, 26 Nov 2025 18:24:18 +0000 Subject: [PATCH 2/3] chore: add feature md Signed-off-by: Eduardo Rodrigues --- .features/pending/disable-write-back-informer.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .features/pending/disable-write-back-informer.md diff --git a/.features/pending/disable-write-back-informer.md b/.features/pending/disable-write-back-informer.md new file mode 100644 index 000000000000..67eda88ce4d1 --- /dev/null +++ b/.features/pending/disable-write-back-informer.md @@ -0,0 +1,7 @@ +Description: Disable write back informer by default +Author: [Eduardo Rodrigues](https://github.com/eduardodbr) +Component: Controller +Issues: 12352 + +Update the controller’s default behavior to disable the write-back informer. We’ve seen several cases of unexpected behavior that appear to be caused by the write-back mechanism, and Kubernetes docs recommend avoiding writes to the informer store. Although turning it off may increase the frequency of 409 Conflict errors, it should help reduce unpredictable controller behavior. + From e05a4d06bec4bb9b56cdf5638ba41ad309d3e3d7 Mon Sep 17 00:00:00 2001 From: Eduardo Rodrigues Date: Fri, 28 Nov 2025 17:39:18 +0000 Subject: [PATCH 3/3] fix: feature component Signed-off-by: Eduardo Rodrigues --- .features/pending/disable-write-back-informer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.features/pending/disable-write-back-informer.md b/.features/pending/disable-write-back-informer.md index 67eda88ce4d1..1627e73edd4a 100644 --- a/.features/pending/disable-write-back-informer.md +++ b/.features/pending/disable-write-back-informer.md @@ -1,6 +1,6 @@ Description: Disable write back informer by default Author: [Eduardo Rodrigues](https://github.com/eduardodbr) -Component: Controller +Component: General Issues: 12352 Update the controller’s default behavior to disable the write-back informer. We’ve seen several cases of unexpected behavior that appear to be caused by the write-back mechanism, and Kubernetes docs recommend avoiding writes to the informer store. Although turning it off may increase the frequency of 409 Conflict errors, it should help reduce unpredictable controller behavior.