Skip to content

Fix checking of MachineCreationTimeout when machine is Pending #1015

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thiyyakat merged 6 commits into from
Jul 25, 2025
Merged

Fix checking of MachineCreationTimeout when machine is Pending #1015

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f4cce7a
Update timeout check: Improve correctness and readability
thiyyakat Jul 14, 2025
4eac994
Change unit tests to reflect updated timeout checking logic
thiyyakat Jul 15, 2025
bee63dd
Address review comments: move variable declaration and add test cases.
thiyyakat Jul 16, 2025
6c1e1d4
Change test case description
thiyyakat Jul 25, 2025
b5b0c4c
Change test case description
thiyyakat Jul 25, 2025
e783289
Change test case description
thiyyakat Jul 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions pkg/util/provider/machinecontroller/machine_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -992,24 +992,24 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph
var (
description string
timeOutDuration time.Duration
timeElapsed time.Duration
)

isMachinePending := machine.Status.CurrentStatus.Phase == v1alpha1.MachinePending
isMachineInPlaceUpdating := machine.Status.CurrentStatus.Phase == v1alpha1.MachineInPlaceUpdating
disableHealthTimeout := machine.Spec.MachineConfiguration != nil && ptr.Deref(machine.Spec.DisableHealthTimeout, false)
sleepTime := 1 * time.Minute

if isMachinePending {
timeOutDuration = c.getEffectiveCreationTimeout(machine).Duration
timeElapsed = metav1.Now().Sub(machine.CreationTimestamp.Time)
} else if isMachineInPlaceUpdating {
timeOutDuration = c.getEffectiveInPlaceUpdateTimeout(machine).Duration
timeElapsed = metav1.Now().Sub(machine.Status.CurrentStatus.LastUpdateTime.Time)
} else {
timeOutDuration = c.getEffectiveHealthTimeout(machine).Duration
timeElapsed = metav1.Now().Sub(machine.Status.CurrentStatus.LastUpdateTime.Time)
}

// Timeout value obtained by subtracting last operation with expected time out period
timeOut := metav1.Now().Add(-timeOutDuration).Sub(machine.Status.CurrentStatus.LastUpdateTime.Time)
if timeOut > 0 {
if timeElapsed > timeOutDuration {
// Machine health timeout occurred while joining or rejoining of machine

if !isMachinePending && !isMachineInPlaceUpdating && !disableHealthTimeout {
Expand Down
38 changes: 35 additions & 3 deletions pkg/util/provider/machinecontroller/machine_util_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2154,13 +2154,45 @@ var _ = Describe("machine_util", func() {
Expect(getErr).To(BeNil())
Expect(data.expect.expectedPhase).To(Equal(updatedTargetMachine.Status.CurrentStatus.Phase))
},
Entry("simple machine with creation Timeout(20 min)", &data{
Entry("Pending machine is marked as Failed when creation timeout (20min) has elapsed", &data{
setup: setup{
machines: []*machinev1.Machine{
newMachine(
&machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)},
&machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending, LastUpdateTime: metav1.NewTime(time.Now().Add(-25 * time.Minute))}},
nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.Now()),
&machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending}},
nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.NewTime(time.Now().Add(-25*time.Minute))),
},
targetMachineName: machineSet1Deploy1 + "-" + "0",
},
expect: expect{
retryPeriod: machineutils.ShortRetry,
err: errSuccessfulPhaseUpdate,
expectedPhase: machinev1.MachineFailed,
},
}),
Entry("Pending machine stays in the Pending phase if creation timeout (20min) has not elapsed", &data{
setup: setup{
machines: []*machinev1.Machine{
newMachine(
&machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)},
&machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending}},
nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.NewTime(time.Now().Add(-15*time.Minute))),
},
targetMachineName: machineSet1Deploy1 + "-" + "0",
},
expect: expect{
retryPeriod: machineutils.LongRetry,
err: nil,
expectedPhase: machinev1.MachinePending,
},
}),
Entry("Pending machine is marked as Failed when creation timeout (20min) has elapsed, even if lastUpdate time is recent", &data{
setup: setup{
machines: []*machinev1.Machine{
newMachine(
&machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)},
&machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending, LastUpdateTime: metav1.NewTime(time.Now().Add(-15 * time.Minute))}},
nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.NewTime(time.Now().Add(-25*time.Minute))),
},
targetMachineName: machineSet1Deploy1 + "-" + "0",
},
Expand Down