diff --git a/alias.go b/alias.go index 72406785ce..35cba30be5 100644 --- a/alias.go +++ b/alias.go @@ -21,7 +21,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client/config" - cfg "sigs.k8s.io/controller-runtime/pkg/config" //nolint:staticcheck + cfg "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -99,8 +99,7 @@ var ( // ConfigFile returns the cfg.File function for deferred config file loading, // this is passed into Options{}.From() to populate the Options fields for // the manager. - // Deprecated: This is deprecated in favor of using Options directly. - ConfigFile = cfg.File //nolint:staticcheck + ConfigFile = cfg.File // NewControllerManagedBy returns a new controller builder that will be started by the provided Manager. NewControllerManagedBy = builder.ControllerManagedBy diff --git a/pkg/builder/controller.go b/pkg/builder/controller.go index f0cfdfc8de..03f9633a74 100644 --- a/pkg/builder/controller.go +++ b/pkg/builder/controller.go @@ -285,7 +285,7 @@ func (blder *Builder) getControllerName(gvk schema.GroupVersionKind, hasGVK bool } func (blder *Builder) doController(r reconcile.Reconciler) error { - globalOpts := blder.mgr.GetControllerOptions() //nolint:staticcheck + globalOpts := blder.mgr.GetControllerOptions() ctrlOptions := blder.ctrlOptions if ctrlOptions.Reconciler == nil { diff --git a/pkg/builder/controller_test.go b/pkg/builder/controller_test.go index 0f78b9150c..782c20ab16 100644 --- a/pkg/builder/controller_test.go +++ b/pkg/builder/controller_test.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -235,7 +235,7 @@ var _ = Describe("application", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{ - Controller: v1alpha1.ControllerConfigurationSpec{ //nolint:staticcheck + Controller: v1alpha1.ControllerConfigurationSpec{ GroupKindConcurrency: map[string]int{ "ReplicaSet.apps": maxConcurrentReconciles, }, diff --git a/pkg/client/apiutil/apimachinery.go b/pkg/client/apiutil/apimachinery.go index 3055f4c4fb..8e2ac48fa2 100644 --- a/pkg/client/apiutil/apimachinery.go +++ b/pkg/client/apiutil/apimachinery.go @@ -95,6 +95,7 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi return gvk, nil } + // Use the given scheme to retrieve all the GVKs for the object. gvks, isUnversioned, err := scheme.ObjectKinds(obj) if err != nil { return schema.GroupVersionKind{}, err @@ -103,16 +104,39 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi return schema.GroupVersionKind{}, fmt.Errorf("cannot create group-version-kind for unversioned type %T", obj) } - if len(gvks) < 1 { - return schema.GroupVersionKind{}, fmt.Errorf("no group-version-kinds associated with type %T", obj) - } - if len(gvks) > 1 { - // this should only trigger for things like metav1.XYZ -- - // normal versioned types should be fine + switch { + case len(gvks) < 1: + // If the object has no GVK, the object might not have been registered with the scheme. + // or it's not a valid object. + return schema.GroupVersionKind{}, fmt.Errorf("no GroupVersionKind associated with Go type %T, was the type registered with the Scheme?", obj) + case len(gvks) > 1: + err := fmt.Errorf("multiple GroupVersionKinds associated with Go type %T within the Scheme, this can happen when a type is registered for multiple GVKs at the same time", obj) + + // We've found multiple GVKs for the object. + currentGVK := obj.GetObjectKind().GroupVersionKind() + if !currentGVK.Empty() { + // If the base object has a GVK, check if it's in the list of GVKs before using it. + for _, gvk := range gvks { + if gvk == currentGVK { + return gvk, nil + } + } + + return schema.GroupVersionKind{}, fmt.Errorf( + "%w: the object's supplied GroupVersionKind %q was not found in the Scheme's list; refusing to guess at one: %q", err, currentGVK, gvks) + } + + // This should only trigger for things like metav1.XYZ -- + // normal versioned types should be fine. + // + // See https://github.com/kubernetes-sigs/controller-runtime/issues/362 + // for more information. return schema.GroupVersionKind{}, fmt.Errorf( - "multiple group-version-kinds associated with type %T, refusing to guess at one", obj) + "%w: callers can either fix their type registration to only register it once, or specify the GroupVersionKind to use for object passed in; refusing to guess at one: %q", err, gvks) + default: + // In any other case, we've found a single GVK for the object. + return gvks[0], nil } - return gvks[0], nil } // RESTClientForGVK constructs a new rest.Interface capable of accessing the resource associated diff --git a/pkg/client/apiutil/lazyrestmapper.go b/pkg/client/apiutil/lazyrestmapper.go index 70c6a11dbc..e9b1e710c2 100644 --- a/pkg/client/apiutil/lazyrestmapper.go +++ b/pkg/client/apiutil/lazyrestmapper.go @@ -33,7 +33,7 @@ type lazyRESTMapper struct { mapper meta.RESTMapper client *discovery.DiscoveryClient knownGroups map[string]*restmapper.APIGroupResources - apiGroups *metav1.APIGroupList + apiGroups []metav1.APIGroup // mutex to provide thread-safe mapper reloading. mu sync.Mutex @@ -45,6 +45,7 @@ func newLazyRESTMapperWithClient(discoveryClient *discovery.DiscoveryClient) (me mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), client: discoveryClient, knownGroups: map[string]*restmapper.APIGroupResources{}, + apiGroups: []metav1.APIGroup{}, }, nil } @@ -147,7 +148,7 @@ func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...st // This operation requires 2 requests: /api and /apis, but only once. For all subsequent calls // this data will be taken from cache. if len(versions) == 0 { - apiGroup, err := m.findAPIGroupByName(groupName) + apiGroup, err := m.findAPIGroupByNameLocked(groupName) if err != nil { return err } @@ -176,11 +177,22 @@ func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...st } // Update information for group resources about the API group by adding new versions. + // Ignore the versions that are already registered. for _, version := range versions { - groupResources.Group.Versions = append(groupResources.Group.Versions, metav1.GroupVersionForDiscovery{ - GroupVersion: metav1.GroupVersion{Group: groupName, Version: version}.String(), - Version: version, - }) + found := false + for _, v := range groupResources.Group.Versions { + if v.Version == version { + found = true + break + } + } + + if !found { + groupResources.Group.Versions = append(groupResources.Group.Versions, metav1.GroupVersionForDiscovery{ + GroupVersion: metav1.GroupVersion{Group: groupName, Version: version}.String(), + Version: version, + }) + } } // Update data in the cache. @@ -197,28 +209,34 @@ func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...st return nil } -// findAPIGroupByName returns API group by its name. -func (m *lazyRESTMapper) findAPIGroupByName(groupName string) (metav1.APIGroup, error) { - // Ensure that required info about existing API groups is received and stored in the mapper. - // It will make 2 API calls to /api and /apis, but only once. - if m.apiGroups == nil { - apiGroups, err := m.client.ServerGroups() - if err != nil { - return metav1.APIGroup{}, fmt.Errorf("failed to get server groups: %w", err) - } - if len(apiGroups.Groups) == 0 { - return metav1.APIGroup{}, fmt.Errorf("received an empty API groups list") +// findAPIGroupByNameLocked returns API group by its name. +func (m *lazyRESTMapper) findAPIGroupByNameLocked(groupName string) (metav1.APIGroup, error) { + // Looking in the cache first. + for _, apiGroup := range m.apiGroups { + if groupName == apiGroup.Name { + return apiGroup, nil } + } - m.apiGroups = apiGroups + // Update the cache if nothing was found. + apiGroups, err := m.client.ServerGroups() + if err != nil { + return metav1.APIGroup{}, fmt.Errorf("failed to get server groups: %w", err) } + if len(apiGroups.Groups) == 0 { + return metav1.APIGroup{}, fmt.Errorf("received an empty API groups list") + } + + m.apiGroups = apiGroups.Groups - for i := range m.apiGroups.Groups { - if groupName == (&m.apiGroups.Groups[i]).Name { - return m.apiGroups.Groups[i], nil + // Looking in the cache again. + for _, apiGroup := range m.apiGroups { + if groupName == apiGroup.Name { + return apiGroup, nil } } + // If there is still nothing, return an error. return metav1.APIGroup{}, fmt.Errorf("failed to find API group %s", groupName) } diff --git a/pkg/client/apiutil/lazyrestmapper_test.go b/pkg/client/apiutil/lazyrestmapper_test.go index 6282370164..f54dbd1600 100644 --- a/pkg/client/apiutil/lazyrestmapper_test.go +++ b/pkg/client/apiutil/lazyrestmapper_test.go @@ -17,14 +17,19 @@ limitations under the License. package apiutil_test import ( + "context" "net/http" "testing" _ "github.com/onsi/ginkgo/v2" gmg "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/envtest" ) @@ -102,38 +107,38 @@ func TestLazyRestMapperProvider(t *testing.T) { // There are no requests before any call g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) - mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "v1") g.Expect(err).NotTo(gmg.HaveOccurred()) g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) - mappings, err := lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}) + mappings, err := lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "v1") g.Expect(err).NotTo(gmg.HaveOccurred()) g.Expect(len(mappings)).To(gmg.Equal(1)) g.Expect(mappings[0].GroupVersionKind.Kind).To(gmg.Equal("pod")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "ingresses"}) g.Expect(err).NotTo(gmg.HaveOccurred()) g.Expect(kind.Kind).To(gmg.Equal("Ingress")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "tokenreviews"}) g.Expect(err).NotTo(gmg.HaveOccurred()) g.Expect(len(kinds)).To(gmg.Equal(1)) g.Expect(kinds[0].Kind).To(gmg.Equal("TokenReview")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) resource, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "priorityclasses"}) g.Expect(err).NotTo(gmg.HaveOccurred()) g.Expect(resource.Resource).To(gmg.Equal("priorityclasses")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(7)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) resources, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "poddisruptionbudgets"}) g.Expect(err).NotTo(gmg.HaveOccurred()) g.Expect(len(resources)).To(gmg.Equal(1)) g.Expect(resources[0].Resource).To(gmg.Equal("poddisruptionbudgets")) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(8)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) }) t.Run("LazyRESTMapper should cache fetched data and doesn't perform any additional requests", func(t *testing.T) { @@ -344,29 +349,29 @@ func TestLazyRestMapperProvider(t *testing.T) { lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, apiutil.WithExperimentalLazyMapper) g.Expect(err).NotTo(gmg.HaveOccurred()) - _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "INVALID"}) + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "INVALID"}, "v1") g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) - _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "INVALID"}) + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "INVALID"}, "v1") g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "INVALID"}) g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "INVALID"}) g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "INVALID"}) g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(7)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "INVALID"}) g.Expect(err).To(gmg.HaveOccurred()) - g.Expect(crt.GetRequestCount()).To(gmg.Equal(8)) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) }) t.Run("LazyRESTMapper should return an error if the version doesn't exist", func(t *testing.T) { @@ -407,4 +412,87 @@ func TestLazyRestMapperProvider(t *testing.T) { g.Expect(err).To(gmg.HaveOccurred()) g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) }) + + t.Run("LazyRESTMapper can fetch CRDs if they were created at runtime", func(t *testing.T) { + g := gmg.NewWithT(t) + + // To fetch all versions mapper does 2 requests: + // GET https://host/api + // GET https://host/apis + // Then, for each version it performs just one request to the API server as usual: + // GET https://host/apis// + + // Note: We have to use a separate restCfg for the Client, otherwise we + // get a race condition on the counting round tripper between the Client + // and the lazy restmapper. + clientRestCfg := rest.CopyConfig(restCfg) + + var crt *countingRoundTripper + restCfg := rest.CopyConfig(restCfg) + restCfg.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { + crt = newCountingRoundTripper(rt) + return crt + } + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each currently registered version: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + s := scheme.Scheme + err = apiextensionsv1.AddToScheme(s) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + c, err := client.New(clientRestCfg, client.Options{Scheme: s}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // Register another CRD in runtime - "riders.crew.example.com". + + crd := &apiextensionsv1.CustomResourceDefinition{} + err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(crd.Spec.Names.Kind).To(gmg.Equal("Driver")) + + newCRD := &apiextensionsv1.CustomResourceDefinition{} + crd.DeepCopyInto(newCRD) + newCRD.Name = "riders.crew.example.com" + newCRD.Spec.Names = apiextensionsv1.CustomResourceDefinitionNames{ + Kind: "Rider", + Plural: "riders", + } + newCRD.ResourceVersion = "" + + // Create the new CRD. + g.Expect(c.Create(context.TODO(), newCRD)).To(gmg.Succeed()) + + // Wait a bit until the CRD is registered. + g.Eventually(func() error { + _, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) + return err + }).Should(gmg.Succeed()) + + // Since we don't specify what version we expect, restmapper will fetch them all and search there. + // To fetch a list of available versions + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each currently registered version: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "rider"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("rider")) + }) } diff --git a/pkg/config/config.go b/pkg/config/config.go index 2caddd1265..8e853d6a0f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -24,24 +24,20 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" ) // ControllerManagerConfiguration defines the functions necessary to parse a config file // and to configure the Options struct for the ctrl.Manager. -// -// Deprecated: This package has been deprecated and will be removed in a future release. type ControllerManagerConfiguration interface { runtime.Object // Complete returns the versioned configuration - Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) //nolint:staticcheck + Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) } // DeferredFileLoader is used to configure the decoder for loading controller // runtime component config types. -// -// Deprecated: This package has been deprecated and will be removed in a future release. type DeferredFileLoader struct { ControllerManagerConfiguration path string @@ -56,8 +52,6 @@ type DeferredFileLoader struct { // Defaults: // * Path: "./config.yaml" // * Kind: GenericControllerManagerConfiguration -// -// Deprecated: This package has been deprecated and will be removed in a future release. func File() *DeferredFileLoader { scheme := runtime.NewScheme() utilruntime.Must(v1alpha1.AddToScheme(scheme)) @@ -69,8 +63,6 @@ func File() *DeferredFileLoader { } // Complete will use sync.Once to set the scheme. -// -// Deprecated: This package has been deprecated and will be removed in a future release. func (d *DeferredFileLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { d.once.Do(d.loadFile) if d.err != nil { @@ -79,33 +71,25 @@ func (d *DeferredFileLoader) Complete() (v1alpha1.ControllerManagerConfiguration return d.ControllerManagerConfiguration.Complete() } -// AtPath will set the path to load the file for the decoder -// -// Deprecated: This package has been deprecated and will be removed in a future release. +// AtPath will set the path to load the file for the decoder. func (d *DeferredFileLoader) AtPath(path string) *DeferredFileLoader { d.path = path return d } // OfKind will set the type to be used for decoding the file into. -// -// Deprecated: This package has been deprecated and will be removed in a future release. func (d *DeferredFileLoader) OfKind(obj ControllerManagerConfiguration) *DeferredFileLoader { d.ControllerManagerConfiguration = obj return d } // InjectScheme will configure the scheme to be used for decoding the file. -// -// Deprecated: This package has been deprecated and will be removed in a future release. func (d *DeferredFileLoader) InjectScheme(scheme *runtime.Scheme) error { d.scheme = scheme return nil } // loadFile is used from the mutex.Once to load the file. -// -// Deprecated: This package has been deprecated and will be removed in a future release. func (d *DeferredFileLoader) loadFile() { if d.scheme == nil { d.err = fmt.Errorf("scheme not supplied to controller configuration loader") diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 71657d8414..3d7aec7e81 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -19,8 +19,8 @@ package config_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/config" //nolint:staticcheck - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" ) var _ = Describe("config", func() { @@ -33,7 +33,7 @@ var _ = Describe("config", func() { }) It("should load a config from file", func() { - conf := v1alpha1.ControllerManagerConfiguration{} //nolint:staticcheck + conf := v1alpha1.ControllerManagerConfiguration{} loader := config.File().AtPath("./testdata/config.yaml").OfKind(&conf) Expect(conf.CacheNamespace).To(Equal("")) diff --git a/pkg/config/doc.go b/pkg/config/doc.go index 176b4edfa1..a169ec5597 100644 --- a/pkg/config/doc.go +++ b/pkg/config/doc.go @@ -22,6 +22,4 @@ limitations under the License. // This uses a deferred file decoding allowing you to chain your configuration // setup. You can pass this into manager.Options#File and it will load your // config. -// -// Deprecated: This package has been deprecated and will be removed in a future release. package config diff --git a/pkg/config/example_test.go b/pkg/config/example_test.go index 44b3113ed7..fb1cd58b5f 100644 --- a/pkg/config/example_test.go +++ b/pkg/config/example_test.go @@ -21,8 +21,9 @@ import ( "os" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/examples/configfile/custom/v1alpha1" - "sigs.k8s.io/controller-runtime/pkg/config" //nolint:staticcheck ) var scheme = runtime.NewScheme() diff --git a/pkg/config/v1alpha1/doc.go b/pkg/config/v1alpha1/doc.go index 9472da2fe0..1e3adbafb8 100644 --- a/pkg/config/v1alpha1/doc.go +++ b/pkg/config/v1alpha1/doc.go @@ -17,6 +17,4 @@ limitations under the License. // Package v1alpha1 provides the ControllerManagerConfiguration used for // configuring ctrl.Manager // +kubebuilder:object:generate=true -// -// Deprecated: This package has been deprecated and will be removed in a future release. package v1alpha1 diff --git a/pkg/config/v1alpha1/register.go b/pkg/config/v1alpha1/register.go index 7a3ec3728e..9efdbc0668 100644 --- a/pkg/config/v1alpha1/register.go +++ b/pkg/config/v1alpha1/register.go @@ -23,18 +23,12 @@ import ( var ( // GroupVersion is group version used to register these objects. - // - // Deprecated: This package has been deprecated and will be removed in a future release. GroupVersion = schema.GroupVersion{Group: "controller-runtime.sigs.k8s.io", Version: "v1alpha1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme. - // - // Deprecated: This package has been deprecated and will be removed in a future release. SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} // AddToScheme adds the types in this group-version to the given scheme. - // - // Deprecated: This package has been deprecated and will be removed in a future release. AddToScheme = SchemeBuilder.AddToScheme ) diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go index 3a3d5d6637..f2226278c6 100644 --- a/pkg/config/v1alpha1/types.go +++ b/pkg/config/v1alpha1/types.go @@ -25,8 +25,6 @@ import ( ) // ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration. -// -// Deprecated: This package has been deprecated and will be removed in a future release. type ControllerManagerConfigurationSpec struct { // SyncPeriod determines the minimum frequency at which watched resources are // reconciled. A lower period will correct entropy more quickly, but reduce @@ -77,8 +75,6 @@ type ControllerManagerConfigurationSpec struct { // ControllerConfigurationSpec defines the global configuration for // controllers registered with the manager. -// -// Deprecated: This package has been deprecated and will be removed in a future release. type ControllerConfigurationSpec struct { // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation // allowed for that controller. @@ -153,20 +149,14 @@ type ControllerWebhook struct { // +kubebuilder:object:root=true // ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API. -// -// Deprecated: This package has been deprecated and will be removed in a future release. type ControllerManagerConfiguration struct { metav1.TypeMeta `json:",inline"` // ControllerManagerConfiguration returns the contfigurations for controllers - // - // Deprecated: This package has been deprecated and will be removed in a future release. ControllerManagerConfigurationSpec `json:",inline"` } // Complete returns the configuration for controller-runtime. -// -// Deprecated: This package has been deprecated and will be removed in a future release. func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) { return *c, nil } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 2c6ab39af7..fe7f94fdc1 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -141,7 +141,7 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller } if options.RecoverPanic == nil { - options.RecoverPanic = mgr.GetControllerOptions().RecoverPanic //nolint:staticcheck + options.RecoverPanic = mgr.GetControllerOptions().RecoverPanic } // Create controller with dependencies set diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 2b56c1858b..b5d816bc28 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -147,7 +147,7 @@ var _ = Describe("controller.Controller", func() { }) It("should default RecoverPanic from the manager", func() { - m, err := manager.New(cfg, manager.Options{Controller: v1alpha1.ControllerConfigurationSpec{RecoverPanic: pointer.Bool(true)}}) //nolint:staticcheck + m, err := manager.New(cfg, manager.Options{Controller: v1alpha1.ControllerConfigurationSpec{RecoverPanic: pointer.Bool(true)}}) Expect(err).NotTo(HaveOccurred()) c, err := controller.New("new-controller", m, controller.Options{ @@ -163,7 +163,7 @@ var _ = Describe("controller.Controller", func() { }) It("should not override RecoverPanic on the controller", func() { - m, err := manager.New(cfg, manager.Options{Controller: v1alpha1.ControllerConfigurationSpec{RecoverPanic: pointer.Bool(true)}}) //nolint:staticcheck + m, err := manager.New(cfg, manager.Options{Controller: v1alpha1.ControllerConfigurationSpec{RecoverPanic: pointer.Bool(true)}}) Expect(err).NotTo(HaveOccurred()) c, err := controller.New("new-controller", m, controller.Options{ diff --git a/pkg/manager/example_test.go b/pkg/manager/example_test.go index 8c5d367082..17557d1817 100644 --- a/pkg/manager/example_test.go +++ b/pkg/manager/example_test.go @@ -22,7 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client/config" - conf "sigs.k8s.io/controller-runtime/pkg/config" //nolint:staticcheck + conf "sigs.k8s.io/controller-runtime/pkg/config" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -92,7 +92,7 @@ func ExampleManager_start() { // using defaults. func ExampleOptions_andFrom() { opts := manager.Options{} - if _, err := opts.AndFrom(conf.File()); err != nil { //nolint:staticcheck + if _, err := opts.AndFrom(conf.File()); err != nil { log.Error(err, "unable to load config") os.Exit(1) } @@ -120,7 +120,7 @@ func ExampleOptions_andFromOrDie() { os.Exit(1) } - mgr, err := manager.New(cfg, manager.Options{}.AndFromOrDie(conf.File())) //nolint:staticcheck + mgr, err := manager.New(cfg, manager.Options{}.AndFromOrDie(conf.File())) if err != nil { log.Error(err, "unable to set up manager") os.Exit(1) diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index f30be8fe18..5ccff8b782 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" @@ -108,7 +108,7 @@ type controllerManager struct { healthzHandler *healthz.Handler // controllerOptions are the global controller options. - controllerOptions v1alpha1.ControllerConfigurationSpec //nolint:staticcheck + controllerOptions v1alpha1.ControllerConfigurationSpec // Logger is the logger that should be used by this manager. // If none is set, it defaults to log.Log global logger. @@ -325,7 +325,7 @@ func (cm *controllerManager) GetLogger() logr.Logger { return cm.logger } -func (cm *controllerManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { //nolint:staticcheck +func (cm *controllerManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { return cm.controllerOptions } @@ -528,7 +528,12 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e // // The shutdown context immediately expires if the gracefulShutdownTimeout is not set. var shutdownCancel context.CancelFunc - cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout) + if cm.gracefulShutdownTimeout < 0 { + // We want to wait forever for the runnables to stop. + cm.shutdownCtx, shutdownCancel = context.WithCancel(context.Background()) + } else { + cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout) + } defer shutdownCancel() // Start draining the errors before acquiring the lock to make sure we don't deadlock diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 9e8d9b7298..2facb1c915 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -36,8 +36,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/config" //nolint:staticcheck - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" @@ -94,11 +94,7 @@ type Manager interface { GetLogger() logr.Logger // GetControllerOptions returns controller global configuration options. - // - // Deprecated: In a future version, the returned value is going to be replaced with a - // different type that doesn't rely on component configuration types. - // This is a temporary warning, and no action is needed as of today. - GetControllerOptions() v1alpha1.ControllerConfigurationSpec //nolint:staticcheck + GetControllerOptions() v1alpha1.ControllerConfigurationSpec } // Options are the arguments for creating a new Manager. @@ -301,11 +297,7 @@ type Options struct { // Controller contains global configuration options for controllers // registered within this manager. // +optional - // - // Deprecated: In a future version, the type of this field is going to be replaced with a - // different struct that doesn't rely on component configuration types. - // This is a temporary warning, and no action is needed as of today. - Controller v1alpha1.ControllerConfigurationSpec //nolint:staticcheck + Controller v1alpha1.ControllerConfigurationSpec // makeBroadcaster allows deferring the creation of the broadcaster to // avoid leaking goroutines if we never call Start on this manager. It also @@ -464,8 +456,6 @@ func New(config *rest.Config, options Options) (Manager, error) { // AndFrom will use a supplied type and convert to Options // any options already set on Options will be ignored, this is used to allow // cli flags to override anything specified in the config file. -// -// Deprecated: This method will be removed in a future release. func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, error) { if inj, wantsScheme := loader.(inject.Scheme); wantsScheme { err := inj.InjectScheme(o.Scheme) @@ -531,8 +521,6 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, } // AndFromOrDie will use options.AndFrom() and will panic if there are errors. -// -// Deprecated: This method will be removed in a future release. func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Options { o, err := o.AndFrom(loader) if err != nil { @@ -541,7 +529,7 @@ func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Opti return o } -func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options { //nolint:staticcheck +func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options { if obj.LeaderElection == nil { // The source does not have any configuration; noop return o diff --git a/pkg/manager/manager_options_test.go b/pkg/manager/manager_options_test.go index 794b1857fb..3718bedcbe 100644 --- a/pkg/manager/manager_options_test.go +++ b/pkg/manager/manager_options_test.go @@ -4,10 +4,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/config" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/config" //nolint:staticcheck - configv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + configv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" ) var _ = Describe("manager.Options", func() { @@ -24,7 +25,7 @@ var _ = Describe("manager.Options", func() { o = Options{Scheme: s} c = customConfig{} - _, err = o.AndFrom(config.File().AtPath("./testdata/custom-config.yaml").OfKind(&c)) //nolint:staticcheck + _, err = o.AndFrom(config.File().AtPath("./testdata/custom-config.yaml").OfKind(&c)) }) It("should not panic or fail", func() { diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 48328db29d..e6358868ad 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -47,7 +47,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" //nolint:staticcheck + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake" @@ -127,8 +127,8 @@ var _ = Describe("manger.Manager", func() { port := int(6090) leaderElect := false - ccfg := &v1alpha1.ControllerManagerConfiguration{ //nolint:staticcheck - ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ //nolint:staticcheck + ccfg := &v1alpha1.ControllerManagerConfiguration{ + ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ SyncPeriod: &duration, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ LeaderElect: &leaderElect, @@ -183,8 +183,8 @@ var _ = Describe("manger.Manager", func() { port := int(6090) leaderElect := false - ccfg := &v1alpha1.ControllerManagerConfiguration{ //nolint:staticcheck - ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ //nolint:staticcheck + ccfg := &v1alpha1.ControllerManagerConfiguration{ + ControllerManagerConfigurationSpec: v1alpha1.ControllerManagerConfigurationSpec{ SyncPeriod: &duration, LeaderElection: &configv1alpha1.LeaderElectionConfiguration{ LeaderElect: &leaderElect, @@ -1076,6 +1076,50 @@ var _ = Describe("manger.Manager", func() { <-runnableStopped }) + It("should wait forever for runnables if gracefulShutdownTimeout is <0 (-1)", func() { + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + m.(*controllerManager).gracefulShutdownTimeout = time.Duration(-1) + + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(100 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(200 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(500 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(1500 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + + ctx, cancel := context.WithCancel(context.Background()) + managerStopDone := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + close(managerStopDone) + }() + <-m.Elected() + cancel() + + beforeDone := time.Now() + <-managerStopDone + Expect(time.Since(beforeDone)).To(BeNumerically(">=", 1500*time.Millisecond)) + }) + } Context("with defaults", func() { @@ -1819,7 +1863,7 @@ type fakeDeferredLoader struct { *v1alpha1.ControllerManagerConfiguration } -func (f *fakeDeferredLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { //nolint:staticcheck +func (f *fakeDeferredLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { return f.ControllerManagerConfiguration.ControllerManagerConfigurationSpec, nil } diff --git a/pkg/metrics/client_go_adapter.go b/pkg/metrics/client_go_adapter.go index dc805a9d04..a8b43ea0a4 100644 --- a/pkg/metrics/client_go_adapter.go +++ b/pkg/metrics/client_go_adapter.go @@ -62,11 +62,44 @@ var ( Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), }, []string{"verb", "url"}) - requestResult = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: RestClientSubsystem, - Name: ResultKey, - Help: "Number of HTTP requests, partitioned by status code, method, and host.", - }, []string{"code", "method", "host"}) + // requestLatency is a Prometheus Histogram metric type partitioned by + // "verb", and "host" labels. It is used for the rest client latency metrics. + requestLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_request_duration_seconds", + Help: "Request latency in seconds. Broken down by verb, and host.", + Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0}, + }, + []string{"verb", "host"}, + ) + + requestSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_request_size_bytes", + Help: "Request size in bytes. Broken down by verb and host.", + // 64 bytes to 16MB + Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216}, + }, + []string{"verb", "host"}, + ) + + responseSize = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_response_size_bytes", + Help: "Response size in bytes. Broken down by verb and host.", + // 64 bytes to 16MB + Buckets: []float64{64, 256, 512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216}, + }, + []string{"verb", "host"}, + ) + + requestResult = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rest_client_requests_total", + Help: "Number of HTTP requests, partitioned by status code, method, and host.", + }, + []string{"code", "method", "host"}, + ) ) func init() { @@ -76,11 +109,17 @@ func init() { // registerClientMetrics sets up the client latency metrics from client-go. func registerClientMetrics() { // register the metrics with our registry + Registry.MustRegister(requestLatency) + Registry.MustRegister(requestSize) + Registry.MustRegister(responseSize) Registry.MustRegister(requestResult) // register the metrics with client-go clientmetrics.Register(clientmetrics.RegisterOpts{ - RequestResult: &resultAdapter{metric: requestResult}, + RequestLatency: &LatencyAdapter{metric: requestLatency}, + RequestSize: &sizeAdapter{metric: requestSize}, + ResponseSize: &sizeAdapter{metric: responseSize}, + RequestResult: &resultAdapter{metric: requestResult}, }) } @@ -102,6 +141,14 @@ func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, late l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds()) } +type sizeAdapter struct { + metric *prometheus.HistogramVec +} + +func (s *sizeAdapter) Observe(ctx context.Context, verb string, host string, size float64) { + s.metric.WithLabelValues(verb, host).Observe(size) +} + type resultAdapter struct { metric *prometheus.CounterVec }