mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 11:21:49 +00:00
fix: reset LaunchTime on instance relaunch to prevent watchdog termination
During cluster restart, restoreInstances resets running instances (with dead QEMU) to pending for relaunch, but didn't reset LaunchTime. The pending watchdog (5min timeout) saw the stale LaunchTime from the original launch and immediately marked instances as failed/terminated, preventing any relaunch from succeeding.
This commit is contained in:
@@ -848,6 +848,13 @@ func (d *Daemon) restoreInstances() {
|
||||
case vm.StateRunning:
|
||||
// Was running but QEMU died - reset to pending so LaunchInstance can transition to running
|
||||
instance.Status = vm.StatePending
|
||||
// Reset LaunchTime so the pending watchdog gives a fresh timeout window.
|
||||
// Without this, the stale LaunchTime from the original launch causes the
|
||||
// watchdog to immediately mark the instance as failed.
|
||||
now := time.Now()
|
||||
if instance.Instance != nil {
|
||||
instance.Instance.LaunchTime = &now
|
||||
}
|
||||
slog.Info("Instance was running but QEMU exited, relaunching", "instance", instance.ID)
|
||||
}
|
||||
|
||||
|
||||
@@ -720,6 +720,39 @@ func TestRestoreInstances_RunningResetToPending(t *testing.T) {
|
||||
"instance should not remain running when QEMU process is dead")
|
||||
}
|
||||
|
||||
// TestRestoreInstances_RunningResetsLaunchTime verifies that when a running
|
||||
// instance is reset to pending for relaunch, its LaunchTime is updated to now.
|
||||
// Without this, the pending watchdog sees the stale LaunchTime and immediately
|
||||
// marks the instance as failed.
|
||||
func TestRestoreInstances_RunningResetsLaunchTime(t *testing.T) {
|
||||
daemon := createDaemonWithJetStream(t)
|
||||
|
||||
staleTime := time.Now().Add(-30 * time.Minute)
|
||||
daemon.Instances.VMS["i-stale-launch"] = &vm.VM{
|
||||
ID: "i-stale-launch",
|
||||
Status: vm.StateRunning,
|
||||
Instance: &ec2.Instance{
|
||||
LaunchTime: &staleTime,
|
||||
},
|
||||
}
|
||||
require.NoError(t, daemon.WriteState())
|
||||
|
||||
daemon.Instances.VMS = make(map[string]*vm.VM)
|
||||
before := time.Now()
|
||||
simulateCleanRestore(t, daemon)
|
||||
|
||||
instance, ok := daemon.Instances.VMS["i-stale-launch"]
|
||||
require.True(t, ok, "instance should be loaded from state")
|
||||
require.NotNil(t, instance.Instance)
|
||||
require.NotNil(t, instance.Instance.LaunchTime)
|
||||
|
||||
// LaunchTime should be reset to approximately now, not the stale 30min-ago time
|
||||
assert.True(t, instance.Instance.LaunchTime.After(before) || instance.Instance.LaunchTime.Equal(before),
|
||||
"LaunchTime should be reset to now, got %v (stale was %v)", *instance.Instance.LaunchTime, staleTime)
|
||||
assert.True(t, time.Since(*instance.Instance.LaunchTime) < pendingWatchdogTimeout,
|
||||
"LaunchTime should be within watchdog timeout window")
|
||||
}
|
||||
|
||||
// TestRestoreInstances_MixedStates verifies recovery with multiple instances
|
||||
// in different states, ensuring each is handled correctly and independently.
|
||||
// Stopped instances (including stopping→stopped transitions) are migrated to shared KV.
|
||||
|
||||
Reference in New Issue
Block a user