From 05006c87aadea4f8716449e655a54eb1f726610c Mon Sep 17 00:00:00 2001 From: Daniel Fan Date: Thu, 1 Aug 2024 14:01:39 -0700 Subject: [PATCH] move cpp config into reconciliation loop Signed-off-by: Daniel Fan --- controllers/bootstrap/init.go | 28 ++-- controllers/commonservice_controller.go | 30 ++-- .../configurationCollector/collector.go | 82 ----------- .../configurationcollector/collector.go | 134 ++++++++++++++++++ controllers/goroutines/cppConfig.go | 70 --------- controllers/goroutines/waitToCreateCsCR.go | 2 +- main.go | 2 - 7 files changed, 168 insertions(+), 180 deletions(-) delete mode 100644 controllers/configurationCollector/collector.go create mode 100644 controllers/configurationcollector/collector.go delete mode 100644 controllers/goroutines/cppConfig.go diff --git a/controllers/bootstrap/init.go b/controllers/bootstrap/init.go index b680d3331..3bbf69986 100644 --- a/controllers/bootstrap/init.go +++ b/controllers/bootstrap/init.go @@ -1654,37 +1654,43 @@ func (b *Bootstrap) PropagateCPPConfig(instance *corev1.ConfigMap) error { // Do not copy ibm-cpp-config in AllNamespace Mode if len(watchNamespaceList) > 1 { - for _, watchNamespace := range watchNamespaceList { - if watchNamespace == instance.Namespace { + for _, ns := range watchNamespaceList { + if ns == instance.Namespace { continue } copiedCPPConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: constant.IBMCPPCONFIG, - Namespace: watchNamespace, + Namespace: ns, + Labels: instance.GetLabels(), }, Data: instance.Data, } if err := b.Client.Create(ctx, copiedCPPConfigMap); err != nil { if errors.IsAlreadyExists(err) { - cmKey := types.NamespacedName{Name: constant.IBMCPPCONFIG, Namespace: watchNamespace} + cmKey := types.NamespacedName{Name: constant.IBMCPPCONFIG, Namespace: ns} existingCM := &corev1.ConfigMap{} - if err := b.Client.Get(ctx, cmKey, existingCM); err != nil { - return fmt.Errorf("failed to get %s ConfigMap in namespace %s: %v", constant.IBMCPPCONFIG, watchNamespace, err) + if err := b.Reader.Get(ctx, cmKey, existingCM); err != nil { + return fmt.Errorf("failed to get %s ConfigMap in namespace %s: %v", constant.IBMCPPCONFIG, ns, err) + } + for k, v := range existingCM.Data { + if _, ok := copiedCPPConfigMap.Data[k]; !ok { + copiedCPPConfigMap.Data[k] = v + } } - if !reflect.DeepEqual(copiedCPPConfigMap.Data, existingCM.Data) { + if !reflect.DeepEqual(copiedCPPConfigMap.Data, existingCM.Data) || !reflect.DeepEqual(copiedCPPConfigMap.Labels, existingCM.Labels) { copiedCPPConfigMap.SetResourceVersion(existingCM.GetResourceVersion()) if err := b.Client.Update(ctx, copiedCPPConfigMap); err != nil { - return fmt.Errorf("failed to update %s ConfigMap in namespace %s: %v", constant.IBMCPPCONFIG, watchNamespace, err) + return fmt.Errorf("failed to update %s ConfigMap in namespace %s: %v", constant.IBMCPPCONFIG, ns, err) } - klog.Infof("Global CPP config %s/%s is updated", watchNamespace, constant.IBMCPPCONFIG) + klog.Infof("Global CPP config %s/%s is updated", ns, constant.IBMCPPCONFIG) } } else { - return fmt.Errorf("failed to create cloned %s ConfigMap in namespace %s: %v", constant.IBMCPPCONFIG, watchNamespace, err) + return fmt.Errorf("failed to create cloned %s ConfigMap in namespace %s: %v", constant.IBMCPPCONFIG, ns, err) } } else { - klog.Infof("Global CPP config %s/%s is propagated to namespace %s", b.CSData.ServicesNs, constant.IBMCPPCONFIG, watchNamespace) + klog.Infof("Global CPP config %s/%s is propagated to namespace %s", b.CSData.ServicesNs, constant.IBMCPPCONFIG, ns) } } } diff --git a/controllers/commonservice_controller.go b/controllers/commonservice_controller.go index 6fa7d1927..ddbcd949c 100644 --- a/controllers/commonservice_controller.go +++ b/controllers/commonservice_controller.go @@ -41,6 +41,7 @@ import ( apiv3 "github.com/IBM/ibm-common-service-operator/api/v3" "github.com/IBM/ibm-common-service-operator/controllers/bootstrap" util "github.com/IBM/ibm-common-service-operator/controllers/common" + "github.com/IBM/ibm-common-service-operator/controllers/configurationcollector" "github.com/IBM/ibm-common-service-operator/controllers/constant" odlm "github.com/IBM/operand-deployment-lifecycle-manager/v4/api/v1alpha1" ) @@ -267,45 +268,46 @@ func (r *CommonServiceReconciler) ReconcileMasterCR(ctx context.Context, instanc instance.Status.Phase = CRFailed } - if statusErr := r.Client.Status().Update(ctx, instance); statusErr != nil { + if statusErr = r.Client.Status().Update(ctx, instance); statusErr != nil { klog.Errorf("Fail to update %s/%s: %v", instance.Namespace, instance.Name, err) return ctrl.Result{}, statusErr } - isEqual, statusErr := r.updateOperandConfig(ctx, newConfigs, serviceControllerMapping) - if statusErr != nil { + var isEqual bool + if isEqual, statusErr = r.updateOperandConfig(ctx, newConfigs, serviceControllerMapping); statusErr != nil { if statusErr := r.updatePhase(ctx, instance, CRFailed); statusErr != nil { klog.Error(statusErr) } klog.Errorf("Fail to reconcile %s/%s: %v", instance.Namespace, instance.Name, statusErr) return ctrl.Result{}, statusErr - } - - // Create Event if there is no update in OperandConfig after applying current CR - if isEqual { + } else if isEqual { r.Recorder.Event(instance, corev1.EventTypeNormal, "Noeffect", fmt.Sprintf("No update, resource sizings in the OperandConfig %s/%s are larger than the profile from CommonService CR %s/%s", r.Bootstrap.CSData.OperatorNs, "common-service", instance.Namespace, instance.Name)) } - isEqual, statusErr = r.updateOperatorConfig(ctx, instance.Spec.OperatorConfigs) - if statusErr != nil { + if isEqual, statusErr = r.updateOperatorConfig(ctx, instance.Spec.OperatorConfigs); statusErr != nil { if statusErr := r.updatePhase(ctx, instance, CRFailed); statusErr != nil { klog.Error(statusErr) } klog.Errorf("Fail to reconcile %s/%s: %v", instance.Namespace, instance.Name, statusErr) return ctrl.Result{}, statusErr + } else if isEqual { + r.Recorder.Event(instance, corev1.EventTypeNormal, "Noeffect", fmt.Sprintf("No update, replica sizings in the OperatorConfig %s/%s are larger than the profile from CommonService CR %s/%s", r.Bootstrap.CSData.OperatorNs, "common-service", instance.Namespace, instance.Name)) } - // Create Event if there is no update in OperandConfig after applying current CR - if isEqual { - r.Recorder.Event(instance, corev1.EventTypeNormal, "Noeffect", fmt.Sprintf("No update, replica sizings in the OperatorConfig %s/%s are larger than the profile from CommonService CR %s/%s", r.Bootstrap.CSData.OperatorNs, "common-service", instance.Namespace, instance.Name)) + if statusErr = configurationcollector.CreateUpdateConfig(r.Bootstrap); statusErr != nil { + if statusErr := r.updatePhase(ctx, instance, CRFailed); statusErr != nil { + klog.Error(statusErr) + } + klog.Errorf("Fail to reconcile %s/%s: %v", instance.Namespace, instance.Name, statusErr) + return ctrl.Result{}, statusErr } - if statusErr := r.Bootstrap.PropagateDefaultCR(instance); statusErr != nil { + if statusErr = r.Bootstrap.PropagateDefaultCR(instance); statusErr != nil { klog.Error(statusErr) return ctrl.Result{}, statusErr } - if statusErr := r.Bootstrap.UpdateResourceLabel(instance); statusErr != nil { + if statusErr = r.Bootstrap.UpdateResourceLabel(instance); statusErr != nil { klog.Error(statusErr) return ctrl.Result{}, statusErr } diff --git a/controllers/configurationCollector/collector.go b/controllers/configurationCollector/collector.go deleted file mode 100644 index f08aee1f7..000000000 --- a/controllers/configurationCollector/collector.go +++ /dev/null @@ -1,82 +0,0 @@ -// -// Copyright 2022 IBM Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package configurationcollector - -import ( - "context" - "strings" - - storagev1 "k8s.io/api/storage/v1" - - "github.com/IBM/ibm-common-service-operator/controllers/bootstrap" -) - -func Buildconfig(config map[string]string, bs *bootstrap.Bootstrap) map[string]string { - builder := configbuilder{data: config, bs: bs} - updatedConfig := builder.setDefaultStorageClass() - return updatedConfig.data -} - -type configbuilder struct { - data map[string]string - bs *bootstrap.Bootstrap -} - -func (b *configbuilder) setDefaultStorageClass() *configbuilder { - scList := &storagev1.StorageClassList{} - err := b.bs.Reader.List(context.TODO(), scList) - if err != nil { - return b - } - if len(scList.Items) == 0 { - return b - } - - var defaultSC string - var defaultSCList []string - var allSCList []string - - for _, sc := range scList.Items { - if defaultSC == "" { - defaultSC = sc.Name - } - if sc.ObjectMeta.GetAnnotations()["storageclass.kubernetes.io/is-default-class"] == "true" { - defaultSCList = append(defaultSCList, sc.Name) - } - if sc.Provisioner == "kubernetes.io/no-provisioner" { - continue - } - allSCList = append(allSCList, sc.GetName()) - } - - if b.data == nil { - b.data = make(map[string]string) - } - if defaultSC != "" { - b.data["storageclass.default"] = defaultSC - } - - if len(defaultSCList) != 1 { - b.data["storageclass.default.list"] = strings.Join(defaultSCList, ",") - } - - if len(allSCList) != 0 { - b.data["storageclass.list"] = strings.Join(allSCList, ",") - } - - return b -} diff --git a/controllers/configurationcollector/collector.go b/controllers/configurationcollector/collector.go new file mode 100644 index 000000000..5de8b034a --- /dev/null +++ b/controllers/configurationcollector/collector.go @@ -0,0 +1,134 @@ +// +// Copyright 2022 IBM Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package configurationcollector + +import ( + "context" + "reflect" + "strings" + + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog" + + "github.com/IBM/ibm-common-service-operator/controllers/bootstrap" + util "github.com/IBM/ibm-common-service-operator/controllers/common" + "github.com/IBM/ibm-common-service-operator/controllers/constant" +) + +func Buildconfig(config map[string]string, bs *bootstrap.Bootstrap) map[string]string { + builder := configbuilder{data: config, bs: bs} + updatedConfig := builder.setDefaultStorageClass() + return updatedConfig.data +} + +type configbuilder struct { + data map[string]string + bs *bootstrap.Bootstrap +} + +func (b *configbuilder) setDefaultStorageClass() *configbuilder { + scList := &storagev1.StorageClassList{} + err := b.bs.Reader.List(context.TODO(), scList) + if err != nil { + return b + } + if len(scList.Items) == 0 { + return b + } + + var defaultSC string + var defaultSCList []string + var allSCList []string + + for _, sc := range scList.Items { + if defaultSC == "" { + defaultSC = sc.Name + } + if sc.ObjectMeta.GetAnnotations()["storageclass.kubernetes.io/is-default-class"] == "true" { + defaultSCList = append(defaultSCList, sc.Name) + } + if sc.Provisioner == "kubernetes.io/no-provisioner" { + continue + } + allSCList = append(allSCList, sc.GetName()) + } + + if b.data == nil { + b.data = make(map[string]string) + } + if defaultSC != "" { + b.data["storageclass.default"] = defaultSC + } + + if len(defaultSCList) != 1 { + b.data["storageclass.default.list"] = strings.Join(defaultSCList, ",") + } + + if len(allSCList) != 0 { + b.data["storageclass.list"] = strings.Join(allSCList, ",") + } + + return b +} + +// CreateUpdateConfig deploys config builder for global cpp configmap +func CreateUpdateConfig(bs *bootstrap.Bootstrap) error { + config := &corev1.ConfigMap{} + if err := bs.Reader.Get(context.TODO(), types.NamespacedName{Name: constant.IBMCPPCONFIG, Namespace: bs.CSData.ServicesNs}, config); err != nil && !errors.IsNotFound(err) { + klog.Errorf("Failed to get ConfigMap %s/%s: %v", bs.CSData.ServicesNs, constant.IBMCPPCONFIG, err) + return err + } else if errors.IsNotFound(err) { + config.ObjectMeta.Name = constant.IBMCPPCONFIG + config.ObjectMeta.Namespace = bs.CSData.ServicesNs + config.Data = make(map[string]string) + config.Data = Buildconfig(config.Data, bs) + if !(config.Labels != nil && config.Labels[constant.CsManagedLabel] == "true") { + util.EnsureLabelsForConfigMap(config, map[string]string{ + constant.CsManagedLabel: "true", + }) + } + if err := bs.Client.Create(context.TODO(), config); err != nil { + klog.Errorf("Failed to create ConfigMap %s/%s: %v", bs.CSData.ServicesNs, constant.IBMCPPCONFIG, err) + return err + } + klog.Infof("Global CPP config %s/%s is created", bs.CSData.ServicesNs, constant.IBMCPPCONFIG) + } else { + orgConfig := config.DeepCopy() + config.Data = Buildconfig(config.Data, bs) + if !(config.Labels != nil && config.Labels[constant.CsManagedLabel] == "true") { + util.EnsureLabelsForConfigMap(config, map[string]string{ + constant.CsManagedLabel: "true", + }) + } + if !reflect.DeepEqual(orgConfig, config) { + if err := bs.Client.Update(context.TODO(), config); err != nil { + klog.Errorf("Failed to update ConfigMap %s/%s: %v", bs.CSData.ServicesNs, constant.IBMCPPCONFIG, err) + return err + } + klog.Infof("Global CPP config %s/%s is updated", bs.CSData.ServicesNs, constant.IBMCPPCONFIG) + } + } + + if err := bs.PropagateCPPConfig(config); err != nil { + klog.Error(err) + return err + } + return nil +} diff --git a/controllers/goroutines/cppConfig.go b/controllers/goroutines/cppConfig.go deleted file mode 100644 index ded7102ff..000000000 --- a/controllers/goroutines/cppConfig.go +++ /dev/null @@ -1,70 +0,0 @@ -// -// Copyright 2022 IBM Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package goroutines - -import ( - "context" - "reflect" - "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" - - "github.com/IBM/ibm-common-service-operator/controllers/bootstrap" - collector "github.com/IBM/ibm-common-service-operator/controllers/configurationCollector" - "github.com/IBM/ibm-common-service-operator/controllers/constant" -) - -// CreateUpdateConfig deploys config builder for global cpp configmap -func CreateUpdateConfig(bs *bootstrap.Bootstrap) { - - for { - config := &corev1.ConfigMap{} - if err := bs.Client.Get(context.TODO(), types.NamespacedName{Name: constant.IBMCPPCONFIG, Namespace: bs.CSData.ServicesNs}, config); err != nil && !errors.IsNotFound(err) { - continue - } else if errors.IsNotFound(err) { - config.ObjectMeta.Name = constant.IBMCPPCONFIG - config.ObjectMeta.Namespace = bs.CSData.ServicesNs - config.Data = make(map[string]string) - config.Data = collector.Buildconfig(config.Data, bs) - if err := bs.Client.Create(context.TODO(), config); err != nil { - time.Sleep(1 * time.Second) - continue - } - klog.Infof("Global CPP config %s/%s is created", bs.CSData.ServicesNs, constant.IBMCPPCONFIG) - } else { - orgConfig := config.DeepCopy() - config.Data = collector.Buildconfig(config.Data, bs) - if !reflect.DeepEqual(orgConfig, config) { - if err := bs.Client.Update(context.TODO(), config); err != nil { - time.Sleep(1 * time.Second) - continue - } - klog.Infof("Global CPP config %s/%s is updated", bs.CSData.ServicesNs, constant.IBMCPPCONFIG) - } - if err := bs.PropagateCPPConfig(config); err != nil { - klog.Error(err) - time.Sleep(1 * time.Second) - continue - } - } - time.Sleep(10 * time.Minute) - } - -} diff --git a/controllers/goroutines/waitToCreateCsCR.go b/controllers/goroutines/waitToCreateCsCR.go index ab254c711..8b2d6e605 100644 --- a/controllers/goroutines/waitToCreateCsCR.go +++ b/controllers/goroutines/waitToCreateCsCR.go @@ -27,7 +27,7 @@ import ( "github.com/IBM/ibm-common-service-operator/controllers/bootstrap" ) -// CreateUpdateConfig deploys config builder for global cpp configmap +// WaitToCreateCsCR waits for the creation of the CommonService CR in the operator namespace. func WaitToCreateCsCR(bs *bootstrap.Bootstrap) { for { klog.Infof("Start to Create CommonService CR in the namespace %s", bs.CSData.OperatorNs) diff --git a/main.go b/main.go index 16c0e5554..789cc6f36 100644 --- a/main.go +++ b/main.go @@ -152,8 +152,6 @@ func main() { klog.Errorf("Cleanup Webhook Resources failed: %v", err) os.Exit(1) } - // Create or Update CPP configuration - go goroutines.CreateUpdateConfig(bs) // Update CS CR Status go goroutines.UpdateCsCrStatus(bs) // Create CS CR