Skip to content

reconcile k8s resource in parallel and with retry logic #1053

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ibm-ci-bot merged 2 commits into from
Jun 5, 2024
+139 −85
Merged

reconcile k8s resource in parallel and with retry logic #1053

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
899b3d2
reconcile k8s resource in parallel and with retry logic
Daniel-Fan Jun 3, 2024
5ee4aba
Separate variables for chunkSize and Retry number
Daniel-Fan Jun 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Separate variables for chunkSize and Retry number 
Signed-off-by: Daniel Fan <[email protected]>
  • Loading branch information
@Daniel-Fan
Daniel-Fan committed Jun 5, 2024
commit 5ee4aba1e1563c3831751cb7f8a2208ed8326ced
3 changes: 3 additions & 0 deletions controllers/constant/constant.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,4 +90,7 @@ const (

//DefaultCSVWaitPeriod is the default period for wait CSV ready
DefaultCSVWaitPeriod = 1 * time.Minute

//DefaultCRRetryNumber is the default maximum number of retry for reconciling a custom resource
DefaultCRRetryNumber = 3
)
12 changes: 6 additions & 6 deletions controllers/operandrequest/reconcile_operand.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ func (r *Reconciler) reconcileCRwithConfig(ctx context.Context, service *operato
go func(res operatorv1alpha1.ConfigResource) {
defer wg.Done()
defer func() { <-semaphore }() // release semaphore
err := r.reconcileK8sResourceWithRetries(ctx, res, service.Name, opConfigName, opConfigNs, chunkSize)
err := r.reconcileK8sResourceWithRetries(ctx, res, service.Name, opConfigName, opConfigNs)
if err != nil {
merr.Add(err)
}
Expand Down Expand Up @@ -486,17 +486,17 @@ func newServiceStatus(operatorName string, namespace string, resources []operato
return serviceSpec
}

func (r *Reconciler) reconcileK8sResourceWithRetries(ctx context.Context, res operatorv1alpha1.ConfigResource, serviceName, opConfigName, opConfigNs string, chunkSize int) error {
func (r *Reconciler) reconcileK8sResourceWithRetries(ctx context.Context, res operatorv1alpha1.ConfigResource, serviceName, opConfigName, opConfigNs string) error {
var err error
for i := 0; i < chunkSize; i++ {
for i := 0; i < int(constant.DefaultCRRetryNumber); i++ {
err = r.reconcileK8sResource(ctx, res, serviceName, opConfigName, opConfigNs)
if err == nil {
return nil
}
klog.Errorf("Failed to reconcile k8s resource %s/%s: %v", res.Namespace, res.Name, err)
if i < chunkSize-1 {
klog.Errorf("Failed to reconcile k8s resource -- Kind: %s, NamespacedName: %s/%s with error: %v", res.Kind, res.Namespace, res.Name, err)
if i < int(constant.DefaultCRRetryNumber)-1 {
waitTime := time.Duration((1 << i) * 4 * int(time.Second))
klog.Warningf("Retry reconcile k8s resource %s/%s after %v", res.Namespace, res.Name, waitTime)
klog.Warningf("Retry reconcile k8s resource -- Kind: %s, NamespacedName: %s/%s after waiting %v", res.Kind, res.Namespace, res.Name, waitTime)
time.Sleep(waitTime)
}
}
Expand Down