Skip to content

Add taint for critical components not ready after update #1017

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aaronfern merged 4 commits into from
Jul 18, 2025
Merged

Add taint for critical components not ready after update #1017

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2cabf18
Add taint for critical components not ready after update
acumino Jul 18, 2025
77c6ca2
Fix spelling
acumino Jul 18, 2025
329f030
Remove PreferNoSchedule taint after inplace update
acumino Jul 18, 2025
8bf1c55
Address Review
acumino Jul 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion pkg/controller/deployment_inplace.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"fmt"
"maps"
"slices"
"sort"

"github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
Expand Down Expand Up @@ -69,7 +70,7 @@ func (dc *controller) rolloutInPlace(ctx context.Context, d *v1alpha1.MachineDep
oldMachineSets, &v1.Taint{
Key: PreferNoScheduleKey,
Value: "True",
Effect: "PreferNoSchedule",
Effect: v1.TaintEffectPreferNoSchedule,
},
)
if err != nil {
Expand Down Expand Up @@ -193,6 +194,18 @@ func (dc *controller) syncMachineSets(ctx context.Context, oldMachineSets []*v1a
// uncordon the node since the inplace update is successful.
node.Spec.Unschedulable = false

// remove the PreferNoSchedule taint if it exists which was added during the inplace update.
node.Spec.Taints = slices.DeleteFunc(node.Spec.Taints, func(t v1.Taint) bool {
return t.Key == PreferNoScheduleKey && t.Value == "True" && t.Effect == v1.TaintEffectPreferNoSchedule
})

// add the critical components not ready taint to the node this is to ensure that
// the pods are not scheduled on the node until the critical components pods are ready.
node.Spec.Taints = append(node.Spec.Taints, v1.Taint{
Key: machineutils.TaintNodeCriticalComponentsNotReady,
Effect: v1.TaintEffectNoSchedule,
})

_, err = dc.targetCoreClient.CoreV1().Nodes().Update(ctx, node, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("failed to remove inplace labels/annotations and uncordon node %s: %w", node.Name, err)
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/provider/machinecontroller/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import (
)

const (
// MCMFinalizerName is the finalizer used to tag dependecies before deletion
// MCMFinalizerName is the finalizer used to tag dependencies before deletion
// of the object. This finalizer is carried over from the MCM
MCMFinalizerName = "machine.sapcloud.io/machine-controller-manager"
// MCFinalizerName is the finalizer created for the external
Expand Down
8 changes: 4 additions & 4 deletions pkg/util/provider/machinecontroller/machine_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -954,8 +954,8 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph
// if the label update successful or failed, then skip the timeout check
if node != nil && metav1.HasLabel(node.ObjectMeta, v1alpha1.LabelKeyNodeUpdateResult) {
if node.Labels[v1alpha1.LabelKeyNodeUpdateResult] == v1alpha1.LabelValueNodeUpdateSuccessful && clone.Status.CurrentStatus.Phase != v1alpha1.MachineInPlaceUpdateSuccessful {
description = fmt.Sprintf("Machine %s successfully updated dependecies", machine.Name)
klog.V(2).Infof("%s with backing node %q and providerID %q sucessfully update the dependecies", description, getNodeName(machine), getProviderID(machine))
description = fmt.Sprintf("Machine %s successfully updated dependencies", machine.Name)
klog.V(2).Infof("%s with backing node %q and providerID %q sucessfully update the dependencies", description, getNodeName(machine), getProviderID(machine))
clone.Status.CurrentStatus = v1alpha1.CurrentStatus{
Phase: v1alpha1.MachineInPlaceUpdateSuccessful,
LastUpdateTime: metav1.Now(),
Expand All @@ -968,8 +968,8 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph
}
cloneDirty = true
} else if node.Labels[v1alpha1.LabelKeyNodeUpdateResult] == v1alpha1.LabelValueNodeUpdateFailed && clone.Status.CurrentStatus.Phase != v1alpha1.MachineInPlaceUpdateFailed {
description = fmt.Sprintf("Machine %s failed to update dependecies: %s", machine.Name, node.Annotations[v1alpha1.AnnotationKeyMachineUpdateFailedReason])
klog.V(2).Infof("%s with backing node %q and providerID %q failed to update dependecies", description, getNodeName(machine), getProviderID(machine))
description = fmt.Sprintf("Machine %s failed to update dependencies: %s", machine.Name, node.Annotations[v1alpha1.AnnotationKeyMachineUpdateFailedReason])
klog.V(2).Infof("%s with backing node %q and providerID %q failed to update dependencies", description, getNodeName(machine), getProviderID(machine))
clone.Status.CurrentStatus = v1alpha1.CurrentStatus{
Phase: v1alpha1.MachineInPlaceUpdateFailed,
LastUpdateTime: metav1.Now(),
Expand Down