pkg/cvo/sync_worker: Consolidate all ClusterOperator errors by reason

pkg/cvo/cvo.go

@@ -281,6 +281,7 @@ func (optr *Operator) InitializeFromPayload(ctx context.Context, restConfig *res
 			Duration: time.Second * 10,
 			Factor:   1.3,
 			Steps:    3,
+			Cap:      time.Second * 15,
 		},
 		optr.exclude,
 		optr.includeTechPreview,

pkg/cvo/cvo_scenarios_test.go

@@ -3554,15 +3554,13 @@ func TestCVO_ParallelError(t *testing.T) {
 	worker := o.configSync.(*SyncWorker)
 	b := &errorResourceBuilder{errors: map[string]error{
 		"0000_10_a_file.yaml": &payload.UpdateError{
-			Reason:       "ClusterOperatorNotAvailable",
 			UpdateEffect: payload.UpdateEffectNone,
-			Name:         "operator-1",
+			Message:      "Failed to reconcile 10-a-file resource",
 		},
 		"0000_20_a_file.yaml": nil,
 		"0000_20_b_file.yaml": &payload.UpdateError{
-			Reason:       "ClusterOperatorNotAvailable",
 			UpdateEffect: payload.UpdateEffectNone,
-			Name:         "operator-2",
+			Message:      "Failed to reconcile 20-b-file resource",
 		},
 	}}
 	worker.builder = b
@@ -3640,9 +3638,6 @@ func TestCVO_ParallelError(t *testing.T) {
 
 	// Step 3: Cancel after we've accumulated 2/3 errors
 	//
-	time.Sleep(100 * time.Millisecond)
-	cancel()
-	//
 	// verify we observe the remaining changes in the first sync
 	for status := range worker.StatusCh() {
 		if status.Failure == nil {
@@ -3675,7 +3670,7 @@ func TestCVO_ParallelError(t *testing.T) {
 		}
 		err := status.Failure
 		uErr, ok := err.(*payload.UpdateError)
-		if !ok || uErr.Reason != "ClusterOperatorsNotAvailable" || uErr.Message != "Some cluster operators are still updating: operator-1, operator-2" {
+		if !ok || uErr.Reason != "MultipleErrors" || uErr.Message != "Multiple errors are preventing progress:\n* Failed to reconcile 10-a-file resource\n* Failed to reconcile 20-b-file resource" {
 			t.Fatalf("unexpected error: %v", err)
 		}
 		if status.LastProgress.IsZero() {
@@ -3707,6 +3702,7 @@ func TestCVO_ParallelError(t *testing.T) {
 		}
 		break
 	}
+	cancel()
 	verifyAllStatus(t, worker.StatusCh())
 
 	client.ClearActions()
@@ -3746,8 +3742,8 @@ func TestCVO_ParallelError(t *testing.T) {
 				{Type: DesiredReleaseAccepted, Status: configv1.ConditionTrue, Reason: "PayloadLoaded",
 					Message: "Payload loaded version=\"1.0.0-abc\" image=\"image/image:1\""},
 				{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse},
-				{Type: ClusterStatusFailing, Status: configv1.ConditionFalse},
-				{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Reason: "ClusterOperatorsNotAvailable", Message: "Working towards 1.0.0-abc: 1 of 3 done (33% complete), waiting on operator-1, operator-2"},
+				{Type: ClusterStatusFailing, Status: configv1.ConditionTrue, Reason: "MultipleErrors", Message: "Multiple errors are preventing progress:\n* Failed to reconcile 10-a-file resource\n* Failed to reconcile 20-b-file resource"},
+				{Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Reason: "MultipleErrors", Message: "Unable to apply 1.0.0-abc: an unknown error has occurred: MultipleErrors"},
 				{Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse},
 			},
 		},

pkg/cvo/internal/operatorstatus.go

@@ -126,22 +126,18 @@ func (b *clusterOperatorBuilder) Do(ctx context.Context) error {
 func checkOperatorHealth(ctx context.Context, client ClusterOperatorsGetter, expected *configv1.ClusterOperator, mode resourcebuilder.Mode) error {
 	if len(expected.Status.Versions) == 0 {
 		return &payload.UpdateError{
-			UpdateEffect: payload.UpdateEffectFail,
-			Reason:       "ClusterOperatorNotAvailable",
-			Message:      fmt.Sprintf("Cluster operator %s does not declare expected versions", expected.Name),
-			Name:         expected.Name,
+			UpdateEffect:        payload.UpdateEffectFail,
+			Reason:              "ClusterOperatorNoVersions",
+			PluralReason:        "ClusterOperatorsNoVersions",
+			Message:             fmt.Sprintf("Cluster operator %s does not declare expected versions", expected.Name),
+			PluralMessageFormat: "Cluster operators %s do not declare expected versions",
+			Name:                expected.Name,
 		}
 	}
 
 	actual, err := client.Get(ctx, expected.Name)
 	if err != nil {
-		return &payload.UpdateError{
-			Nested:       err,
-			UpdateEffect: payload.UpdateEffectNone,
-			Reason:       "ClusterOperatorNotAvailable",
-			Message:      fmt.Sprintf("Cluster operator %s has not yet reported success", expected.Name),
-			Name:         expected.Name,
-		}
+		return err
 	}
 
 	// undone is a sorted slice of transition messages for incomplete operands.
@@ -195,11 +191,13 @@ func checkOperatorHealth(ctx context.Context, client ClusterOperatorsGetter, exp
 			nestedMessage = fmt.Errorf("cluster operator %s is %s=%s: %s: %s", actual.Name, availableCondition.Type, availableCondition.Status, availableCondition.Reason, availableCondition.Message)
 		}
 		return &payload.UpdateError{
-			Nested:       nestedMessage,
-			UpdateEffect: payload.UpdateEffectFail,
-			Reason:       "ClusterOperatorNotAvailable",
-			Message:      fmt.Sprintf("Cluster operator %s is not available", actual.Name),
-			Name:         actual.Name,
+			Nested:              nestedMessage,
+			UpdateEffect:        payload.UpdateEffectFail,
+			Reason:              "ClusterOperatorNotAvailable",
+			PluralReason:        "ClusterOperatorsNotAvailable",
+			Message:             fmt.Sprintf("Cluster operator %s is not available", actual.Name),
+			PluralMessageFormat: "Cluster operators %s are not available",
+			Name:                actual.Name,
 		}
 	}
 
@@ -209,23 +207,27 @@ func checkOperatorHealth(ctx context.Context, client ClusterOperatorsGetter, exp
 			nestedMessage = fmt.Errorf("cluster operator %s is %s=%s: %s, %s", actual.Name, degradedCondition.Type, degradedCondition.Status, degradedCondition.Reason, degradedCondition.Message)
 		}
 		return &payload.UpdateError{
-			Nested:       nestedMessage,
-			UpdateEffect: payload.UpdateEffectFailAfterInterval,
-			Reason:       "ClusterOperatorDegraded",
-			Message:      fmt.Sprintf("Cluster operator %s is degraded", actual.Name),
-			Name:         actual.Name,
+			Nested:              nestedMessage,
+			UpdateEffect:        payload.UpdateEffectFailAfterInterval,
+			Reason:              "ClusterOperatorDegraded",
+			PluralReason:        "ClusterOperatorsDegraded",
+			Message:             fmt.Sprintf("Cluster operator %s is degraded", actual.Name),
+			PluralMessageFormat: "Cluster operators %s are degraded",
+			Name:                actual.Name,
 		}
 	}
 
 	// during initialization we allow undone versions
 	if len(undone) > 0 && mode != resourcebuilder.InitializingMode {
 		nestedMessage = fmt.Errorf("cluster operator %s is available and not degraded but has not finished updating to target version", actual.Name)
 		return &payload.UpdateError{
-			Nested:       nestedMessage,
-			UpdateEffect: payload.UpdateEffectNone,
-			Reason:       "ClusterOperatorUpdating",
-			Message:      fmt.Sprintf("Cluster operator %s is updating versions", actual.Name),
-			Name:         actual.Name,
+			Nested:              nestedMessage,
+			UpdateEffect:        payload.UpdateEffectNone,
+			Reason:              "ClusterOperatorUpdating",
+			PluralReason:        "ClusterOperatorsUpdating",
+			Message:             fmt.Sprintf("Cluster operator %s is updating versions", actual.Name),
+			PluralMessageFormat: "Cluster operators %s are updating versions",
+			Name:                actual.Name,
 		}
 	}