From 61e8eb9d919b72ef94031bb85ab5b202f5ef359b Mon Sep 17 00:00:00 2001
From: Julian Sommer <115516918+juliansommer@users.noreply.github.com>
Date: Wed, 25 Feb 2026 10:48:16 +1000
Subject: [PATCH] fix: reset LaunchTime on instance relaunch to prevent
 watchdog termination

During cluster restart, restoreInstances resets running instances (with
dead QEMU) to pending for relaunch, but didn't reset LaunchTime. The
pending watchdog (5min timeout) saw the stale LaunchTime from the
original launch and immediately marked instances as failed/terminated,
preventing any relaunch from succeeding.
---
 hive/daemon/daemon.go     |  7 +++++++
 hive/daemon/state_test.go | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/hive/daemon/daemon.go b/hive/daemon/daemon.go
index 1b18ec9d2..6eca534c4 100644
--- a/hive/daemon/daemon.go
+++ b/hive/daemon/daemon.go
@@ -848,6 +848,13 @@ func (d *Daemon) restoreInstances() {
 		case vm.StateRunning:
 			// Was running but QEMU died - reset to pending so LaunchInstance can transition to running
 			instance.Status = vm.StatePending
+			// Reset LaunchTime so the pending watchdog gives a fresh timeout window.
+			// Without this, the stale LaunchTime from the original launch causes the
+			// watchdog to immediately mark the instance as failed.
+			now := time.Now()
+			if instance.Instance != nil {
+				instance.Instance.LaunchTime = &now
+			}
 			slog.Info("Instance was running but QEMU exited, relaunching", "instance", instance.ID)
 		}
 
diff --git a/hive/daemon/state_test.go b/hive/daemon/state_test.go
index a3d46ab66..40fe42fb3 100644
--- a/hive/daemon/state_test.go
+++ b/hive/daemon/state_test.go
@@ -720,6 +720,39 @@ func TestRestoreInstances_RunningResetToPending(t *testing.T) {
 		"instance should not remain running when QEMU process is dead")
 }
 
+// TestRestoreInstances_RunningResetsLaunchTime verifies that when a running
+// instance is reset to pending for relaunch, its LaunchTime is updated to now.
+// Without this, the pending watchdog sees the stale LaunchTime and immediately
+// marks the instance as failed.
+func TestRestoreInstances_RunningResetsLaunchTime(t *testing.T) {
+	daemon := createDaemonWithJetStream(t)
+
+	staleTime := time.Now().Add(-30 * time.Minute)
+	daemon.Instances.VMS["i-stale-launch"] = &vm.VM{
+		ID:     "i-stale-launch",
+		Status: vm.StateRunning,
+		Instance: &ec2.Instance{
+			LaunchTime: &staleTime,
+		},
+	}
+	require.NoError(t, daemon.WriteState())
+
+	daemon.Instances.VMS = make(map[string]*vm.VM)
+	before := time.Now()
+	simulateCleanRestore(t, daemon)
+
+	instance, ok := daemon.Instances.VMS["i-stale-launch"]
+	require.True(t, ok, "instance should be loaded from state")
+	require.NotNil(t, instance.Instance)
+	require.NotNil(t, instance.Instance.LaunchTime)
+
+	// LaunchTime should be reset to approximately now, not the stale 30min-ago time
+	assert.True(t, instance.Instance.LaunchTime.After(before) || instance.Instance.LaunchTime.Equal(before),
+		"LaunchTime should be reset to now, got %v (stale was %v)", *instance.Instance.LaunchTime, staleTime)
+	assert.True(t, time.Since(*instance.Instance.LaunchTime) < pendingWatchdogTimeout,
+		"LaunchTime should be within watchdog timeout window")
+}
+
 // TestRestoreInstances_MixedStates verifies recovery with multiple instances
 // in different states, ensuring each is handled correctly and independently.
 // Stopped instances (including stopping→stopped transitions) are migrated to shared KV.