Skip to content

OCPBUGS-75931: [release-4.20] fix(cpo): Correct route labeling logic for HCP router infrastructure #7642

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
csrwng wants to merge 1 commit into
base: release-4.20
Choose a base branch
from
+2,061 −212
Open

OCPBUGS-75931: [release-4.20] fix(cpo): Correct route labeling logic for HCP router infrastructure #7642

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 67 additions & 40 deletions control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1094,13 +1094,17 @@ func (r *HostedControlPlaneReconciler) reconcileCPOV2(ctx context.Context, hcp *
return fmt.Errorf("failed to reconcile metrics config: %w", err)
}

if useHCPRouter(hcp) {
if routerv2.UseHCPRouter(hcp) {
if err := r.admitHCPManagedRoutes(ctx, hcp, infraStatus.InternalHCPRouterHost, infraStatus.ExternalHCPRouterHost); err != nil {
return fmt.Errorf("failed to admit HCP managed routes: %w", err)
}
if err := r.cleanupOldRouterResources(ctx, hcp); err != nil {
return fmt.Errorf("failed to cleanup old router resources: %w", err)
}
} else {
if err := r.removeHCPIngressFromRoutes(ctx, hcp); err != nil {
return fmt.Errorf("failed to remove HCP ingress from routes: %w", err)
}
}

if _, exists := hcp.Annotations[hyperv1.DisableIgnitionServerAnnotation]; !exists {
Expand Down Expand Up @@ -1160,22 +1164,6 @@ func (r *HostedControlPlaneReconciler) reconcileCPOV2(ctx context.Context, hcp *
return utilerrors.NewAggregate(errs)
}

// useHCPRouter returns true if a dedicated common router is created for a HCP to handle ingress for the managed endpoints.
// This is true when the API input specifies intent for the following:
// 1 - AWS endpointAccess is private somehow (i.e. publicAndPrivate or private) or is public and configured with external DNS.
// 2 - When 1 is true, we recommend (and automate via CLI) ServicePublishingStrategy to be "Route" for all endpoints but the KAS
// which needs a dedicated Service type LB external to be exposed if no external DNS is supported.
// Otherwise, the Routes use the management cluster Domain and resolve through the default ingress controller.
func useHCPRouter(hostedControlPlane *hyperv1.HostedControlPlane) bool {
if sharedingress.UseSharedIngress() {
return false
}
if hostedControlPlane.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
return false
}
return util.IsPrivateHCP(hostedControlPlane) || util.IsPublicWithDNS(hostedControlPlane)
}

func IsStorageAndCSIManaged(hostedControlPlane *hyperv1.HostedControlPlane) bool {
if hostedControlPlane.Spec.Platform.Type == hyperv1.IBMCloudPlatform || hostedControlPlane.Spec.Platform.Type == hyperv1.PowerVSPlatform {
return false
Expand Down Expand Up @@ -1327,7 +1315,7 @@ func (r *HostedControlPlaneReconciler) reconcileKonnectivityServerService(ctx co
if serviceStrategy.Route != nil {
hostname = serviceStrategy.Route.Hostname
}
return kas.ReconcileKonnectivityExternalRoute(konnectivityRoute, p.OwnerRef, hostname, r.DefaultIngressDomain, util.UseDedicatedDNSForKonnectivity(hcp))
return kas.ReconcileKonnectivityExternalRoute(konnectivityRoute, p.OwnerRef, hostname, r.DefaultIngressDomain, util.LabelHCPRoutes(hcp))
}); err != nil {
return fmt.Errorf("failed to reconcile Konnectivity server external route: %w", err)
}
Expand Down Expand Up @@ -1365,7 +1353,7 @@ func (r *HostedControlPlaneReconciler) reconcileOAuthServerService(ctx context.C
if serviceStrategy.Route != nil {
hostname = serviceStrategy.Route.Hostname
}
return oauth.ReconcileExternalPublicRoute(oauthExternalPublicRoute, p.OwnerRef, hostname, r.DefaultIngressDomain, util.UseDedicatedDNSForOAuth(hcp))
return oauth.ReconcileExternalPublicRoute(oauthExternalPublicRoute, p.OwnerRef, hostname, r.DefaultIngressDomain, util.LabelHCPRoutes(hcp))
}); err != nil {
return fmt.Errorf("failed to reconcile OAuth external public route: %w", err)
}
Expand All @@ -1379,7 +1367,7 @@ func (r *HostedControlPlaneReconciler) reconcileOAuthServerService(ctx context.C
// Reconcile the external private route if a hostname is specified
if serviceStrategy.Route != nil && serviceStrategy.Route.Hostname != "" {
if _, err := createOrUpdate(ctx, r.Client, oauthExternalPrivateRoute, func() error {
return oauth.ReconcileExternalPrivateRoute(oauthExternalPrivateRoute, p.OwnerRef, serviceStrategy.Route.Hostname, r.DefaultIngressDomain, util.UseDedicatedDNSForOAuth(hcp))
return oauth.ReconcileExternalPrivateRoute(oauthExternalPrivateRoute, p.OwnerRef, serviceStrategy.Route.Hostname, r.DefaultIngressDomain, util.LabelHCPRoutes(hcp))
}); err != nil {
return fmt.Errorf("failed to reconcile OAuth external private route: %w", err)
}
Expand Down Expand Up @@ -1437,41 +1425,43 @@ func (r *HostedControlPlaneReconciler) reconcileOLMPackageServerService(ctx cont
}

func (r *HostedControlPlaneReconciler) reconcileHCPRouterServices(ctx context.Context, hcp *hyperv1.HostedControlPlane, createOrUpdate upsert.CreateOrUpdateFN) error {
if sharedingress.UseSharedIngress() || hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
pubSvc := manifests.RouterPublicService(hcp.Namespace)
privSvc := manifests.PrivateRouterService(hcp.Namespace)
if !routerv2.UseHCPRouter(hcp) {
if _, err := util.DeleteIfNeeded(ctx, r.Client, pubSvc); err != nil {
return fmt.Errorf("failed to delete public router service: %w", err)
}
if _, err := util.DeleteIfNeeded(ctx, r.Client, privSvc); err != nil {
return fmt.Errorf("failed to delete private router service: %w", err)
}
return nil
}

// Create the Service type LB internal for private endpoints.
pubSvc := manifests.RouterPublicService(hcp.Namespace)
if util.IsPrivateHCP(hcp) {
svc := manifests.PrivateRouterService(hcp.Namespace)
if _, err := createOrUpdate(ctx, r.Client, svc, func() error {
return ingress.ReconcileRouterService(svc, true, true, hcp)
if _, err := createOrUpdate(ctx, r.Client, privSvc, func() error {
return ingress.ReconcileRouterService(privSvc, true, true, hcp)
}); err != nil {
return fmt.Errorf("failed to reconcile private router service: %w", err)
}
if !util.IsPublicHCP(hcp) {
// Remove the public router Service if it exists
err := r.Get(ctx, client.ObjectKeyFromObject(pubSvc), pubSvc)
if err != nil {
if !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to check whether public router service exists: %w", err)
}
} else {
if err := r.Delete(ctx, pubSvc); err != nil {
return fmt.Errorf("failed to delete public router service: %w", err)
}
}
} else {
if _, err := util.DeleteIfNeeded(ctx, r.Client, privSvc); err != nil {
return fmt.Errorf("failed to delete private router service: %w", err)
}
}

// When Public access endpoint we need to create a Service type LB external.
if util.IsPublicWithDNS(hcp) {
// When Public access endpoint AND routes use HCP router, create a Service type LB external.
// This ensures we only create public router infrastructure when routes are labeled for it.
if util.IsPublicHCP(hcp) && util.LabelHCPRoutes(hcp) {
if _, err := createOrUpdate(ctx, r.Client, pubSvc, func() error {
return ingress.ReconcileRouterService(pubSvc, false, util.IsPrivateHCP(hcp), hcp)
}); err != nil {
return fmt.Errorf("failed to reconcile router service: %w", err)
}
} else {
if _, err := util.DeleteIfNeeded(ctx, r.Client, pubSvc); err != nil {
return fmt.Errorf("failed to delete public router service: %w", err)
}
}
return nil
}
Expand Down Expand Up @@ -1577,7 +1567,7 @@ func (r *HostedControlPlaneReconciler) reconcileInternalRouterServiceStatus(ctx
}

func (r *HostedControlPlaneReconciler) reconcileExternalRouterServiceStatus(ctx context.Context, hcp *hyperv1.HostedControlPlane) (host string, needed bool, message string, err error) {
if !util.IsPublicWithDNS(hcp) || sharedingress.UseSharedIngress() || hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
if !util.IsPublicHCP(hcp) || !util.LabelHCPRoutes(hcp) || sharedingress.UseSharedIngress() || hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
return
}
return r.reconcileRouterServiceStatus(ctx, manifests.RouterPublicService(hcp.Namespace), events.NewMessageCollector(ctx, r.Client))
Expand Down Expand Up @@ -2572,6 +2562,43 @@ func (r *HostedControlPlaneReconciler) admitHCPManagedRoutes(ctx context.Context
return nil
}

func (r *HostedControlPlaneReconciler) removeHCPIngressFromRoutes(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
routeList := &routev1.RouteList{}
if err := r.List(ctx, routeList, client.InNamespace(hcp.Namespace)); err != nil {
return fmt.Errorf("failed to list routes: %w", err)
}

for i := range routeList.Items {
route := &routeList.Items[i]
if _, hasHCPLabel := route.Labels[util.HCPRouteLabel]; hasHCPLabel {
// Skip routes that should be managed by the HCP router
continue
}
// Only clear ingress entries that correspond to router named "router".
// This matches the RouterName used by ReconcileRouteStatus in ingress/router.go
// when the HCP router admits routes. We filter by this specific name to ensure
// we only remove ingress entries from the HCP router, not from other routers
// (e.g., the default ingress controller router).
originalRoute := route.DeepCopy()
filteredIngress := make([]routev1.RouteIngress, 0, len(route.Status.Ingress))
for _, ingress := range route.Status.Ingress {
if ingress.RouterName != "router" {
filteredIngress = append(filteredIngress, ingress)
}
}
if len(filteredIngress) != len(route.Status.Ingress) {
route.Status.Ingress = filteredIngress
if !equality.Semantic.DeepEqual(originalRoute.Status, route.Status) {
if err := r.Status().Patch(ctx, route, client.MergeFrom(originalRoute)); err != nil {
return fmt.Errorf("failed to clear route %s ingress: %w", route.Name, err)
}
}
}
}

return nil
}

func (r *HostedControlPlaneReconciler) cleanupOldRouterResources(ctx context.Context, hcp *hyperv1.HostedControlPlane) error {
oldRouterResources := []client.Object{
&appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: hcp.Namespace, Name: "private-router"}},
Expand Down
Loading