From 0ff03be0dd73d7f17165edfa7a2b3512a069c9f9 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Wed, 11 Feb 2026 15:10:55 +1000 Subject: [PATCH] fix --- hive/daemon/daemon.go | 30 +++++++++++-------- hive/daemon/daemon_handlers.go | 4 +-- hive/handlers/ec2/image/service_impl.go | 40 +++++++++++-------------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/hive/daemon/daemon.go b/hive/daemon/daemon.go index 7e8521399..d8cd42fbd 100644 --- a/hive/daemon/daemon.go +++ b/hive/daemon/daemon.go @@ -643,20 +643,21 @@ func (d *Daemon) restoreInstances() { continue } - // QEMU is not running - handle transitional states from interrupted operations + // QEMU is not running -- resolve transitional states from interrupted operations switch instance.Status { - case vm.StateStopping: - instance.Status = vm.StateStopped - slog.Info("Instance was stopping, QEMU exited, marking stopped", "instance", instance.ID) - if err := d.WriteState(); err != nil { - slog.Error("Failed to write state", "instance", instance.ID, "error", err) + case vm.StateStopping, vm.StateShuttingDown: + prevStatus := instance.Status + if instance.Status == vm.StateStopping { + instance.Status = vm.StateStopped + } else { + instance.Status = vm.StateTerminated } - continue - case vm.StateShuttingDown: - instance.Status = vm.StateTerminated - slog.Info("Instance was shutting-down, QEMU exited, marking terminated", "instance", instance.ID) + slog.Info("QEMU exited during transition, finalizing state", + "instance", instance.ID, "from", prevStatus, "to", instance.Status) if err := d.WriteState(); err != nil { - slog.Error("Failed to write state", "instance", instance.ID, "error", err) + slog.Error("Failed to persist state, will retry on next restart", + "instance", instance.ID, "error", err) + instance.Status = prevStatus // revert so next restart retries } continue case vm.StateRunning: @@ -697,6 +698,11 @@ func (d *Daemon) reconnectInstance(instance *vm.VM) error { sub, err := d.natsConn.QueueSubscribe(fmt.Sprintf("ec2.cmd.%s", instance.ID), "hive-events", d.handleEC2Events) if err != nil { d.mu.Unlock() + // Clean up the QMP connection we just opened + if instance.QMPClient != nil && instance.QMPClient.Conn != nil { + instance.QMPClient.Conn.Close() + instance.QMPClient = nil + } return fmt.Errorf("failed to subscribe to NATS: %w", err) } d.natsSubscriptions[instance.ID] = sub @@ -705,7 +711,7 @@ func (d *Daemon) reconnectInstance(instance *vm.VM) error { instance.Status = vm.StateRunning if err := d.WriteState(); err != nil { - slog.Error("Failed to persist reconnected instance state", "instanceId", instance.ID, "err", err) + return fmt.Errorf("failed to persist reconnected instance state: %w", err) } slog.Info("Successfully reconnected to running instance", "instance", instance.ID) diff --git a/hive/daemon/daemon_handlers.go b/hive/daemon/daemon_handlers.go index 777186321..54915f141 100644 --- a/hive/daemon/daemon_handlers.go +++ b/hive/daemon/daemon_handlers.go @@ -874,13 +874,13 @@ func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) { d.Instances.Mu.Unlock() if !ok { - slog.Error("CreateImage: instance not found", "instanceId", instanceID) + slog.Warn("CreateImage: instance not found", "instanceId", instanceID) respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) return } if status != vm.StateRunning && status != vm.StateStopped { - slog.Error("CreateImage: instance not in valid state", "instanceId", instanceID, "status", status) + slog.Warn("CreateImage: instance not in valid state", "instanceId", instanceID, "status", status) respondWithError(awserrors.ErrorIncorrectInstanceState) return } diff --git a/hive/handlers/ec2/image/service_impl.go b/hive/handlers/ec2/image/service_impl.go index 2d924f137..86a17e888 100644 --- a/hive/handlers/ec2/image/service_impl.go +++ b/hive/handlers/ec2/image/service_impl.go @@ -253,15 +253,13 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams) (*e "rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID, "isRunning", params.IsRunning) - // Step 1: Snapshot the root volume + // Step 1: Snapshot root volume (live via NATS or offline from S3) + snapshotFn := s.snapshotStoppedVolume if params.IsRunning { - if err := s.snapshotRunningVolume(params.RootVolumeID, snapshotID); err != nil { - return nil, err - } - } else { - if err := s.snapshotStoppedVolume(params.RootVolumeID, snapshotID); err != nil { - return nil, err - } + snapshotFn = s.snapshotRunningVolume + } + if err := snapshotFn(params.RootVolumeID, snapshotID); err != nil { + return nil, err } // Step 2: Read source volume config for size @@ -273,7 +271,11 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams) (*e volumeSizeGiB := volumeConfig.VolumeMetadata.SizeGiB // Step 3: Read source AMI config for architecture, platform, etc. - var sourceAMI viperblock.AMIMetadata + sourceAMI := viperblock.AMIMetadata{ + Architecture: "x86_64", + PlatformDetails: "Linux/UNIX", + Virtualization: "hvm", + } if params.SourceImageID != "" { srcCfg, err := s.getAMIConfig(params.SourceImageID) if err != nil { @@ -281,12 +283,6 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams) (*e return nil, errors.New(awserrors.ErrorServerInternal) } sourceAMI = srcCfg - } else { - sourceAMI = viperblock.AMIMetadata{ - Architecture: "x86_64", - PlatformDetails: "Linux/UNIX", - Virtualization: "hvm", - } } // Step 4: Store snapshot metadata @@ -422,18 +418,18 @@ func (s *ImageServiceImpl) snapshotStoppedVolume(volumeID, snapshotID string) er return errors.New(awserrors.ErrorServerInternal) } + // Clean up local WAL files regardless of snapshot success or failure + defer func() { + if err := vb.RemoveLocalFiles(); err != nil { + slog.Error("snapshotStoppedVolume: failed to remove local files", "volumeId", volumeID, "err", err) + } + }() + if _, err := vb.CreateSnapshot(snapshotID); err != nil { slog.Error("snapshotStoppedVolume: CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err) - if cleanupErr := vb.RemoveLocalFiles(); cleanupErr != nil { - slog.Error("snapshotStoppedVolume: failed to remove local files after snapshot failure", "volumeId", volumeID, "err", cleanupErr) - } return errors.New(awserrors.ErrorServerInternal) } - if err := vb.RemoveLocalFiles(); err != nil { - slog.Error("snapshotStoppedVolume: failed to remove local files", "volumeId", volumeID, "err", err) - } - slog.Info("snapshotStoppedVolume: snapshot created", "volumeId", volumeID, "snapshotId", snapshotID) return nil }