diff --git a/hive/daemon/daemon.go b/hive/daemon/daemon.go index 1b18ec9d2..6eca534c4 100644 --- a/hive/daemon/daemon.go +++ b/hive/daemon/daemon.go @@ -848,6 +848,13 @@ func (d *Daemon) restoreInstances() { case vm.StateRunning: // Was running but QEMU died - reset to pending so LaunchInstance can transition to running instance.Status = vm.StatePending + // Reset LaunchTime so the pending watchdog gives a fresh timeout window. + // Without this, the stale LaunchTime from the original launch causes the + // watchdog to immediately mark the instance as failed. + now := time.Now() + if instance.Instance != nil { + instance.Instance.LaunchTime = &now + } slog.Info("Instance was running but QEMU exited, relaunching", "instance", instance.ID) } diff --git a/hive/daemon/state_test.go b/hive/daemon/state_test.go index a3d46ab66..40fe42fb3 100644 --- a/hive/daemon/state_test.go +++ b/hive/daemon/state_test.go @@ -720,6 +720,39 @@ func TestRestoreInstances_RunningResetToPending(t *testing.T) { "instance should not remain running when QEMU process is dead") } +// TestRestoreInstances_RunningResetsLaunchTime verifies that when a running +// instance is reset to pending for relaunch, its LaunchTime is updated to now. +// Without this, the pending watchdog sees the stale LaunchTime and immediately +// marks the instance as failed. +func TestRestoreInstances_RunningResetsLaunchTime(t *testing.T) { + daemon := createDaemonWithJetStream(t) + + staleTime := time.Now().Add(-30 * time.Minute) + daemon.Instances.VMS["i-stale-launch"] = &vm.VM{ + ID: "i-stale-launch", + Status: vm.StateRunning, + Instance: &ec2.Instance{ + LaunchTime: &staleTime, + }, + } + require.NoError(t, daemon.WriteState()) + + daemon.Instances.VMS = make(map[string]*vm.VM) + before := time.Now() + simulateCleanRestore(t, daemon) + + instance, ok := daemon.Instances.VMS["i-stale-launch"] + require.True(t, ok, "instance should be loaded from state") + require.NotNil(t, instance.Instance) + require.NotNil(t, instance.Instance.LaunchTime) + + // LaunchTime should be reset to approximately now, not the stale 30min-ago time + assert.True(t, instance.Instance.LaunchTime.After(before) || instance.Instance.LaunchTime.Equal(before), + "LaunchTime should be reset to now, got %v (stale was %v)", *instance.Instance.LaunchTime, staleTime) + assert.True(t, time.Since(*instance.Instance.LaunchTime) < pendingWatchdogTimeout, + "LaunchTime should be within watchdog timeout window") +} + // TestRestoreInstances_MixedStates verifies recovery with multiple instances // in different states, ensuring each is handled correctly and independently. // Stopped instances (including stopping→stopped transitions) are migrated to shared KV.