diff --git a/pkg/util/provider/machinecontroller/machine_util.go b/pkg/util/provider/machinecontroller/machine_util.go index 29125113a..46e011e0a 100644 --- a/pkg/util/provider/machinecontroller/machine_util.go +++ b/pkg/util/provider/machinecontroller/machine_util.go @@ -992,24 +992,24 @@ func (c *controller) reconcileMachineHealth(ctx context.Context, machine *v1alph var ( description string timeOutDuration time.Duration + timeElapsed time.Duration ) isMachinePending := machine.Status.CurrentStatus.Phase == v1alpha1.MachinePending isMachineInPlaceUpdating := machine.Status.CurrentStatus.Phase == v1alpha1.MachineInPlaceUpdating disableHealthTimeout := machine.Spec.MachineConfiguration != nil && ptr.Deref(machine.Spec.DisableHealthTimeout, false) sleepTime := 1 * time.Minute - if isMachinePending { timeOutDuration = c.getEffectiveCreationTimeout(machine).Duration + timeElapsed = metav1.Now().Sub(machine.CreationTimestamp.Time) } else if isMachineInPlaceUpdating { timeOutDuration = c.getEffectiveInPlaceUpdateTimeout(machine).Duration + timeElapsed = metav1.Now().Sub(machine.Status.CurrentStatus.LastUpdateTime.Time) } else { timeOutDuration = c.getEffectiveHealthTimeout(machine).Duration + timeElapsed = metav1.Now().Sub(machine.Status.CurrentStatus.LastUpdateTime.Time) } - - // Timeout value obtained by subtracting last operation with expected time out period - timeOut := metav1.Now().Add(-timeOutDuration).Sub(machine.Status.CurrentStatus.LastUpdateTime.Time) - if timeOut > 0 { + if timeElapsed > timeOutDuration { // Machine health timeout occurred while joining or rejoining of machine if !isMachinePending && !isMachineInPlaceUpdating && !disableHealthTimeout { diff --git a/pkg/util/provider/machinecontroller/machine_util_test.go b/pkg/util/provider/machinecontroller/machine_util_test.go index f4f29932a..26d928a9b 100644 --- a/pkg/util/provider/machinecontroller/machine_util_test.go +++ b/pkg/util/provider/machinecontroller/machine_util_test.go @@ -2154,13 +2154,45 @@ var _ = Describe("machine_util", func() { Expect(getErr).To(BeNil()) Expect(data.expect.expectedPhase).To(Equal(updatedTargetMachine.Status.CurrentStatus.Phase)) }, - Entry("simple machine with creation Timeout(20 min)", &data{ + Entry("Pending machine is marked as Failed when creation timeout (20min) has elapsed", &data{ setup: setup{ machines: []*machinev1.Machine{ newMachine( &machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)}, - &machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending, LastUpdateTime: metav1.NewTime(time.Now().Add(-25 * time.Minute))}}, - nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.Now()), + &machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending}}, + nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.NewTime(time.Now().Add(-25*time.Minute))), + }, + targetMachineName: machineSet1Deploy1 + "-" + "0", + }, + expect: expect{ + retryPeriod: machineutils.ShortRetry, + err: errSuccessfulPhaseUpdate, + expectedPhase: machinev1.MachineFailed, + }, + }), + Entry("Pending machine stays in the Pending phase if creation timeout (20min) has not elapsed", &data{ + setup: setup{ + machines: []*machinev1.Machine{ + newMachine( + &machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)}, + &machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending}}, + nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.NewTime(time.Now().Add(-15*time.Minute))), + }, + targetMachineName: machineSet1Deploy1 + "-" + "0", + }, + expect: expect{ + retryPeriod: machineutils.LongRetry, + err: nil, + expectedPhase: machinev1.MachinePending, + }, + }), + Entry("Pending machine is marked as Failed when creation timeout (20min) has elapsed, even if lastUpdate time is recent", &data{ + setup: setup{ + machines: []*machinev1.Machine{ + newMachine( + &machinev1.MachineTemplateSpec{ObjectMeta: *newObjectMeta(&metav1.ObjectMeta{GenerateName: machineSet1Deploy1}, 0)}, + &machinev1.MachineStatus{CurrentStatus: machinev1.CurrentStatus{Phase: machinev1.MachinePending, LastUpdateTime: metav1.NewTime(time.Now().Add(-15 * time.Minute))}}, + nil, nil, map[string]string{machinev1.NodeLabelKey: "node-0-0"}, true, metav1.NewTime(time.Now().Add(-25*time.Minute))), }, targetMachineName: machineSet1Deploy1 + "-" + "0", },