From deae613a75bfc7ddc2f042dca83bea9a7291c2f2 Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Tue, 24 Feb 2026 14:28:24 +1000 Subject: [PATCH] refactor: split daemon_handlers.go into per-domain files Split the monolithic daemon_handlers.go into focused files: - daemon_handlers_account.go - daemon_handlers_console.go - daemon_handlers_igw.go - daemon_handlers_image.go - daemon_handlers_instance.go - daemon_handlers_key.go - daemon_handlers_snapshot.go - daemon_handlers_tag.go - daemon_handlers_volume.go - daemon_handlers_vpc.go Add tests for ModifyVolume success, TerminateStoppedInstance with volumes, DescribeInstanceTypes capacity filter, StartStoppedInstance type-not-available, AttachVolume/DetachVolume validation paths, RunInstances error paths, and DescribeInstances malformed ID. Fix diff-coverage.sh to use git rename/copy detection (-M -C) so file splits are not incorrectly flagged as all-new code. --- .github/workflows/e2e.yml | 1 - hive/daemon/daemon_handlers.go | 1858 +------------------- hive/daemon/daemon_handlers_account.go | 29 + hive/daemon/daemon_handlers_console.go | 123 ++ hive/daemon/daemon_handlers_igw.go | 37 + hive/daemon/daemon_handlers_image.go | 110 ++ hive/daemon/daemon_handlers_instance.go | 1036 +++++++++++ hive/daemon/daemon_handlers_key.go | 21 + hive/daemon/daemon_handlers_snapshot.go | 21 + hive/daemon/daemon_handlers_tag.go | 17 + hive/daemon/daemon_handlers_test.go | 736 ++++++++ hive/daemon/daemon_handlers_volume.go | 492 ++++++ hive/daemon/daemon_handlers_vpc.go | 41 + hive/daemon/daemon_test.go | 18 - hive/handlers/ec2/igw/service_nats_test.go | 8 +- hive/handlers/ec2/vpc/service_nats_test.go | 16 +- scripts/diff-coverage.sh | 6 +- 17 files changed, 2680 insertions(+), 1890 deletions(-) create mode 100644 hive/daemon/daemon_handlers_account.go create mode 100644 hive/daemon/daemon_handlers_console.go create mode 100644 hive/daemon/daemon_handlers_igw.go create mode 100644 hive/daemon/daemon_handlers_image.go create mode 100644 hive/daemon/daemon_handlers_instance.go create mode 100644 hive/daemon/daemon_handlers_key.go create mode 100644 hive/daemon/daemon_handlers_snapshot.go create mode 100644 hive/daemon/daemon_handlers_tag.go create mode 100644 hive/daemon/daemon_handlers_volume.go create mode 100644 hive/daemon/daemon_handlers_vpc.go diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 40147d394..779acd4ef 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -7,7 +7,6 @@ on: - dev paths: - "**/*.go" - - "!**/*_test.go" - go.mod - go.sum - Makefile diff --git a/hive/daemon/daemon_handlers.go b/hive/daemon/daemon_handlers.go index c461b158e..0b04e28af 100644 --- a/hive/daemon/daemon_handlers.go +++ b/hive/daemon/daemon_handlers.go @@ -1,21 +1,13 @@ package daemon import ( - "encoding/base64" "encoding/json" - "fmt" "log/slog" "net" - "os" - "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/hive/hive/awserrors" "github.com/mulgadc/hive/hive/config" - gateway_ec2_instance "github.com/mulgadc/hive/hive/gateway/ec2/instance" - handlers_ec2_image "github.com/mulgadc/hive/hive/handlers/ec2/image" "github.com/mulgadc/hive/hive/qmp" "github.com/mulgadc/hive/hive/utils" "github.com/mulgadc/hive/hive/vm" @@ -93,558 +85,6 @@ func (d *Daemon) handleEC2Events(msg *nats.Msg) { } } -// handleAttachVolume performs a three-phase hot-plug: -// -// Phase 1: ebs.mount via NATS (rolls back with ebs.unmount) -// Phase 2: QMP blockdev-add (rolls back Phase 1) -// Phase 3: QMP device_add (rolls back Phase 2 + Phase 1) -func (d *Daemon) handleAttachVolume(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { - slog.Info("Attaching volume to instance", "instanceId", command.ID) - - // Validate AttachVolumeData - if command.AttachVolumeData == nil || command.AttachVolumeData.VolumeID == "" { - slog.Error("AttachVolume: missing attach volume data") - respondWithError(awserrors.ErrorInvalidParameterValue) - return - } - - volumeID := command.AttachVolumeData.VolumeID - device := command.AttachVolumeData.Device - - // Validate instance is running - d.Instances.Mu.Lock() - status := instance.Status - d.Instances.Mu.Unlock() - - if status != vm.StateRunning { - slog.Error("AttachVolume: instance not running", "instanceId", command.ID, "status", status) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - // Validate volume exists and is available - volCfg, err := d.volumeService.GetVolumeConfig(volumeID) - if err != nil { - slog.Error("AttachVolume: failed to get volume config", "volumeId", volumeID, "err", err) - respondWithError(awserrors.ErrorInvalidVolumeNotFound) - return - } - - if volCfg.VolumeMetadata.State != "available" { - slog.Error("AttachVolume: volume not available", "volumeId", volumeID, "state", volCfg.VolumeMetadata.State) - respondWithError(awserrors.ErrorVolumeInUse) - return - } - - // Check AZ compatibility — volume and instance must be in the same AZ - if volCfg.VolumeMetadata.AvailabilityZone != "" && d.config.AZ != "" && - volCfg.VolumeMetadata.AvailabilityZone != d.config.AZ { - slog.Error("AttachVolume: volume and instance are in different AZs", - "volumeId", volumeID, "volumeAZ", volCfg.VolumeMetadata.AvailabilityZone, - "instanceAZ", d.config.AZ) - respondWithError(awserrors.ErrorInvalidVolumeZoneMismatch) - return - } - - // Determine device name - if device == "" { - d.Instances.Mu.Lock() - device = nextAvailableDevice(instance) - d.Instances.Mu.Unlock() - if device == "" { - slog.Error("AttachVolume: no available device names") - respondWithError(awserrors.ErrorAttachmentLimitExceeded) - return - } - } - - // Create EBS mount request - ebsRequest := config.EBSRequest{ - Name: volumeID, - DeviceName: device, - } - - ebsMountData, err := json.Marshal(ebsRequest) - if err != nil { - slog.Error("AttachVolume: failed to marshal ebs.mount request", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - mountReply, err := d.natsConn.Request(d.ebsTopic("mount"), ebsMountData, 30*time.Second) - if err != nil { - slog.Error("AttachVolume: ebs.mount failed", "volumeId", volumeID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - var mountResp config.EBSMountResponse - if err := json.Unmarshal(mountReply.Data, &mountResp); err != nil { - slog.Error("AttachVolume: failed to unmarshal mount response", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - if mountResp.Error != "" { - slog.Error("AttachVolume: mount returned error", "volumeId", volumeID, "err", mountResp.Error) - respondWithError(awserrors.ErrorServerInternal) - return - } - - nbdURI := mountResp.URI - if nbdURI == "" { - slog.Error("AttachVolume: mount response has empty URI", "volumeId", volumeID) - d.rollbackEBSMount(ebsRequest) - respondWithError(awserrors.ErrorServerInternal) - return - } - ebsRequest.NBDURI = nbdURI - - // Parse NBDURI for QMP blockdev-add - serverType, socketPath, nbdHost, nbdPort, err := utils.ParseNBDURI(nbdURI) - if err != nil { - slog.Error("AttachVolume: failed to parse NBDURI", "uri", nbdURI, "err", err) - // Rollback: unmount - d.rollbackEBSMount(ebsRequest) - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Build QMP server argument - var serverArg map[string]any - if serverType == "unix" { - serverArg = map[string]any{"type": "unix", "path": socketPath} - } else { - serverArg = map[string]any{"type": "inet", "host": nbdHost, "port": fmt.Sprintf("%d", nbdPort)} - } - - nodeName := fmt.Sprintf("nbd-%s", volumeID) - deviceID := fmt.Sprintf("vdisk-%s", volumeID) - iothreadID := fmt.Sprintf("ioth-%s", volumeID) - - // QMP object-add: create iothread for this volume - iothreadCmd := qmp.QMPCommand{ - Execute: "object-add", - Arguments: map[string]any{ - "qom-type": "iothread", - "id": iothreadID, - }, - } - _, err = d.SendQMPCommand(instance.QMPClient, iothreadCmd, instance.ID) - if err != nil { - slog.Error("AttachVolume: QMP object-add iothread failed", "volumeId", volumeID, "err", err) - d.rollbackEBSMount(ebsRequest) - respondWithError(awserrors.ErrorServerInternal) - return - } - - // QMP blockdev-add - blockdevCmd := qmp.QMPCommand{ - Execute: "blockdev-add", - Arguments: map[string]any{ - "node-name": nodeName, - "driver": "nbd", - "server": serverArg, - "export": "", - "read-only": false, - }, - } - - _, err = d.SendQMPCommand(instance.QMPClient, blockdevCmd, instance.ID) - if err != nil { - slog.Error("AttachVolume: QMP blockdev-add failed", "volumeId", volumeID, "err", err) - d.rollbackEBSMount(ebsRequest) - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Determine which hotplug root port to use based on device letter. - // /dev/sdf -> hotplug1, /dev/sdg -> hotplug2, etc. - hotplugBus := "" - if len(device) > 0 { - letter := device[len(device)-1] - if letter >= 'f' && letter <= 'p' { - hotplugBus = fmt.Sprintf("hotplug%d", letter-'f'+1) - } - } - - // QMP device_add - deviceAddArgs := map[string]any{ - "driver": "virtio-blk-pci", - "id": deviceID, - "drive": nodeName, - "iothread": iothreadID, - } - if hotplugBus != "" { - deviceAddArgs["bus"] = hotplugBus - } - deviceAddCmd := qmp.QMPCommand{ - Execute: "device_add", - Arguments: deviceAddArgs, - } - - _, err = d.SendQMPCommand(instance.QMPClient, deviceAddCmd, instance.ID) - if err != nil { - slog.Error("AttachVolume: QMP device_add failed, rolling back blockdev", "volumeId", volumeID, "err", err) - // Rollback: blockdev-del then unmount - if _, delErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ - Execute: "blockdev-del", - Arguments: map[string]any{"node-name": nodeName}, - }, instance.ID); delErr != nil { - slog.Error("AttachVolume: rollback blockdev-del failed, skipping EBS unmount", "volumeId", volumeID, "err", delErr) - } else { - d.rollbackEBSMount(ebsRequest) - } - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Update instance state: replace existing entry for this volume (handles - // stop/start cycles that keep stale entries) or append a new one. - instance.EBSRequests.Mu.Lock() - replaced := false - for i, req := range instance.EBSRequests.Requests { - if req.Name == volumeID { - instance.EBSRequests.Requests[i] = ebsRequest - replaced = true - break - } - } - if !replaced { - instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, ebsRequest) - } - instance.EBSRequests.Mu.Unlock() - - // Update BlockDeviceMappings on the ec2.Instance - d.Instances.Mu.Lock() - if instance.Instance != nil { - now := time.Now() - mapping := &ec2.InstanceBlockDeviceMapping{} - mapping.SetDeviceName(device) - mapping.Ebs = &ec2.EbsInstanceBlockDevice{} - mapping.Ebs.SetVolumeId(volumeID) - mapping.Ebs.SetAttachTime(now) - mapping.Ebs.SetDeleteOnTermination(false) - mapping.Ebs.SetStatus("attached") - instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping) - } - d.Instances.Mu.Unlock() - - // Update volume metadata in S3 - if err := d.volumeService.UpdateVolumeState(volumeID, "in-use", command.ID, device); err != nil { - slog.Error("AttachVolume: failed to update volume metadata", "volumeId", volumeID, "err", err) - } - - // Persist state - if err := d.WriteState(); err != nil { - slog.Error("AttachVolume: failed to write state", "err", err) - } - - d.respondWithVolumeAttachment(msg, respondWithError, volumeID, command.ID, device, "attached") - slog.Info("Volume attached successfully", "volumeId", volumeID, "instanceId", command.ID, "device", device) -} - -// handleDetachVolume performs a three-phase hot-unplug (reverse of attach): -// -// Phase 1: QMP device_del (remove guest device) -// Phase 2: QMP blockdev-del (remove block node) -// Phase 3: ebs.unmount NATS (stop NBD server) -func (d *Daemon) handleDetachVolume(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { - slog.Info("Detaching volume from instance", "instanceId", command.ID) - - // Validate DetachVolumeData - if command.DetachVolumeData == nil || command.DetachVolumeData.VolumeID == "" { - slog.Error("DetachVolume: missing detach volume data") - respondWithError(awserrors.ErrorInvalidParameterValue) - return - } - - volumeID := command.DetachVolumeData.VolumeID - device := command.DetachVolumeData.Device - force := command.DetachVolumeData.Force - - // Validate instance is running - d.Instances.Mu.Lock() - status := instance.Status - d.Instances.Mu.Unlock() - - if status != vm.StateRunning { - slog.Error("DetachVolume: instance not running", "instanceId", command.ID, "status", status) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - // Find the volume in EBSRequests - instance.EBSRequests.Mu.Lock() - var ebsReq config.EBSRequest - found := false - for _, req := range instance.EBSRequests.Requests { - if req.Name == volumeID { - ebsReq = req - found = true - break - } - } - instance.EBSRequests.Mu.Unlock() - - if !found { - slog.Error("DetachVolume: volume not attached to instance", "volumeId", volumeID, "instanceId", command.ID) - respondWithError(awserrors.ErrorIncorrectState) - return - } - - // Reject detaching boot/EFI/CloudInit volumes - if ebsReq.Boot || ebsReq.EFI || ebsReq.CloudInit { - slog.Error("DetachVolume: cannot detach boot/EFI/CloudInit volume", "volumeId", volumeID) - respondWithError(awserrors.ErrorOperationNotPermitted) - return - } - - // Optional device cross-check - if device != "" && ebsReq.DeviceName != "" && device != ebsReq.DeviceName { - slog.Error("DetachVolume: device mismatch", "requested", device, "actual", ebsReq.DeviceName) - respondWithError(awserrors.ErrorInvalidParameterValue) - return - } - - deviceID := fmt.Sprintf("vdisk-%s", volumeID) - nodeName := fmt.Sprintf("nbd-%s", volumeID) - iothreadID := fmt.Sprintf("ioth-%s", volumeID) - - // Phase 1: QMP device_del (remove guest device) - _, err := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ - Execute: "device_del", - Arguments: map[string]any{"id": deviceID}, - }, instance.ID) - if err != nil { - if !force { - slog.Error("DetachVolume: QMP device_del failed", "volumeId", volumeID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - slog.Warn("DetachVolume: QMP device_del failed (force=true, continuing)", "volumeId", volumeID, "err", err) - } - - // Brief pause for guest to acknowledge PCI removal - time.Sleep(d.detachDelay) - - // Phase 2: QMP blockdev-del (remove block node) - _, blockdevErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ - Execute: "blockdev-del", - Arguments: map[string]any{"node-name": nodeName}, - }, instance.ID) - if blockdevErr != nil { - // Block node still referenced by QEMU; do not clean up state or unmount — - // tearing down the NBD server would crash the VM, and removing metadata - // would allow the volume to be double-attached. - slog.Error("DetachVolume: QMP blockdev-del failed, leaving volume state intact", "volumeId", volumeID, "err", blockdevErr) - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Phase 2b: QMP object-del (remove iothread, best-effort) - _, iothreadErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ - Execute: "object-del", - Arguments: map[string]any{"id": iothreadID}, - }, instance.ID) - if iothreadErr != nil { - slog.Warn("DetachVolume: QMP object-del iothread failed (non-fatal)", "volumeId", volumeID, "err", iothreadErr) - } - - // Phase 3: ebs.unmount via NATS (best-effort) - d.rollbackEBSMount(ebsReq) - - // State cleanup: remove volume from EBSRequests (search by name to avoid stale index) - instance.EBSRequests.Mu.Lock() - for i, req := range instance.EBSRequests.Requests { - if req.Name == volumeID { - instance.EBSRequests.Requests = append(instance.EBSRequests.Requests[:i], instance.EBSRequests.Requests[i+1:]...) - break - } - } - instance.EBSRequests.Mu.Unlock() - - // Remove from BlockDeviceMappings - d.Instances.Mu.Lock() - if instance.Instance != nil { - filtered := make([]*ec2.InstanceBlockDeviceMapping, 0, len(instance.Instance.BlockDeviceMappings)) - for _, bdm := range instance.Instance.BlockDeviceMappings { - if bdm.Ebs != nil && bdm.Ebs.VolumeId != nil && *bdm.Ebs.VolumeId == volumeID { - continue - } - filtered = append(filtered, bdm) - } - instance.Instance.BlockDeviceMappings = filtered - } - d.Instances.Mu.Unlock() - - // Update volume metadata to "available" - if err := d.volumeService.UpdateVolumeState(volumeID, "available", "", ""); err != nil { - slog.Error("DetachVolume: failed to update volume metadata", "volumeId", volumeID, "err", err) - } - - // Persist state - if err := d.WriteState(); err != nil { - slog.Error("DetachVolume: failed to write state", "err", err) - } - - d.respondWithVolumeAttachment(msg, respondWithError, volumeID, command.ID, ebsReq.DeviceName, "detaching") - slog.Info("Volume detached successfully", "volumeId", volumeID, "instanceId", command.ID) -} - -func (d *Daemon) handleStartInstance(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { - slog.Info("Starting instance", "id", command.ID) - - // Validate instance is in stopped state - d.Instances.Mu.Lock() - status := instance.Status - d.Instances.Mu.Unlock() - - if status != vm.StateStopped { - slog.Error("StartInstance: instance not in stopped state", "instanceId", command.ID, "status", status) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - // Allocate resources - instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType] - if ok { - if err := d.resourceMgr.allocate(instanceType); err != nil { - slog.Error("Failed to allocate resources for start command", "id", command.ID, "err", err) - respondWithError(awserrors.ErrorInsufficientInstanceCapacity) - return - } - } - - // Clear stop attribute before launch so WriteState inside LaunchInstance - // persists the correct attributes. Without this, a daemon restart after - // a stop→start cycle would see StopInstance=true and skip reconnecting QEMU. - d.Instances.Mu.Lock() - instance.Attributes = command.Attributes - d.Instances.Mu.Unlock() - - // Launch the instance infrastructure (QEMU, QMP, NATS subscriptions) - err := d.LaunchInstance(instance) - - if err != nil { - slog.Error("handleEC2RunInstances LaunchInstance failed", "err", err) - // Free the resource on failure - if ok { - d.resourceMgr.deallocate(instanceType) - } - respondWithError(awserrors.ErrorServerInternal) - return - } - - slog.Info("Instance started", "instanceId", instance.ID) - - if err := msg.Respond(fmt.Appendf(nil, `{"status":"running","instanceId":"%s"}`, instance.ID)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } -} - -func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { - isTerminate := command.Attributes.TerminateInstance - action := "Stopping" - initialState := vm.StateStopping - finalState := vm.StateStopped - if isTerminate { - action = "Terminating" - initialState = vm.StateShuttingDown - finalState = vm.StateTerminated - } - - slog.Info(action+" instance", "id", command.ID) - - // Check state validity before attempting transition — return the correct - // AWS error code when the instance is already stopped/terminated/etc. - d.Instances.Mu.Lock() - currentState := instance.Status - d.Instances.Mu.Unlock() - if !vm.IsValidTransition(currentState, initialState) { - slog.Warn("Instance in incorrect state for "+strings.ToLower(action), - "instanceId", instance.ID, "currentState", string(currentState)) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - // Transition to the initial transitional state - if err := d.TransitionState(instance, initialState); err != nil { - slog.Error("Failed to transition to "+string(initialState), "instanceId", instance.ID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Respond immediately - operation will complete in background - // stopInstance() handles the QMP shutdown command, so we don't send it here - if err := msg.Respond([]byte(`{}`)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - // Run cleanup in goroutine to not block NATS - go func(inst *vm.VM, attrs qmp.Attributes) { - stopErr := d.stopInstance(map[string]*vm.VM{inst.ID: inst}, isTerminate) - - if stopErr != nil { - slog.Error("Failed to "+strings.ToLower(action)+" instance", "err", stopErr, "id", inst.ID) - if err := d.TransitionState(inst, vm.StateError); err != nil { - slog.Error("Failed to transition to error state", "instanceId", inst.ID, "err", err) - } - } else { - d.Instances.Mu.Lock() - inst.Attributes = attrs - inst.LastNode = d.node - d.Instances.Mu.Unlock() - - if err := d.TransitionState(inst, finalState); err != nil { - slog.Error("Failed to transition to final state", "instanceId", inst.ID, "err", err) - } - slog.Info("Instance "+string(finalState), "id", inst.ID) - - // For stop (not terminate): release ownership to shared KV so any - // daemon can pick up the next StartInstance request. - if !isTerminate && d.jsManager != nil { - - // Write to shared KV first — if daemon crashes after this but - // before local cleanup, restoreInstances handles the overlap. - if err := d.jsManager.WriteStoppedInstance(inst.ID, inst); err != nil { - slog.Error("Failed to write stopped instance to shared KV, keeping local ownership", - "instanceId", inst.ID, "err", err) - } else { - // Unsubscribe from per-instance NATS topic - d.mu.Lock() - if sub, ok := d.natsSubscriptions[inst.ID]; ok { - if err := sub.Unsubscribe(); err != nil { - slog.Error("Failed to unsubscribe stopped instance", "instanceId", inst.ID, "err", err) - } - delete(d.natsSubscriptions, inst.ID) - } - d.mu.Unlock() - - // Remove from local instance map - d.Instances.Mu.Lock() - delete(d.Instances.VMS, inst.ID) - d.Instances.Mu.Unlock() - - // Persist local state without the stopped instance - if err := d.WriteState(); err != nil { - slog.Error("Failed to persist state after releasing stopped instance, re-adding to local map for consistency", - "instanceId", inst.ID, "err", err) - // Re-add to local map so in-memory state matches on-disk state. - // On next restart, restoreInstances will retry the migration. - d.Instances.Mu.Lock() - d.Instances.VMS[inst.ID] = inst - d.Instances.Mu.Unlock() - } else { - slog.Info("Released stopped instance ownership to shared KV", - "instanceId", inst.ID, "lastNode", d.node) - } - } - } - } - }(instance, command.Attributes) -} - func (d *Daemon) handleQMPCommand(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { resp, err := d.SendQMPCommand(instance.QMPClient, command.QMPCommand, instance.ID) if err != nil { @@ -687,869 +127,7 @@ func (d *Daemon) handleQMPCommand(msg *nats.Msg, command qmp.Command, instance * } } -// handleEC2RunInstances processes incoming EC2 RunInstances requests -func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) { - slog.Debug("Received message on subject", "subject", msg.Subject) - slog.Debug("Message data", "data", string(msg.Data)) - - // Initialize runInstancesInput before unmarshaling into it - runInstancesInput := &ec2.RunInstancesInput{} - var errResp []byte - - errResp = utils.UnmarshalJsonPayload(runInstancesInput, msg.Data) - - if errResp != nil { - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - slog.Error("Request does not match RunInstancesInput") - return - } - - // Validate inputs - err := gateway_ec2_instance.ValidateRunInstancesInput(runInstancesInput) - - if err != nil { - slog.Error("handleEC2RunInstances validation failed", "err", awserrors.ErrorValidationError) - errResp = utils.GenerateErrorPayload(awserrors.ErrorValidationError) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - - } - - slog.Info("Processing RunInstances request for instance type", "instanceType", *runInstancesInput.InstanceType) - - // Check if instance type is supported - instanceType, exists := d.resourceMgr.instanceTypes[*runInstancesInput.InstanceType] - if !exists { - slog.Error("handleEC2RunInstances instance lookup", "err", awserrors.ErrorInvalidInstanceType, "InstanceType", *runInstancesInput.InstanceType) - errResp = utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceType) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - // Validate AMI exists before allocating resources - if runInstancesInput.ImageId != nil { - _, err := d.imageService.GetAMIConfig(*runInstancesInput.ImageId) - if err != nil { - slog.Error("handleEC2RunInstances AMI not found", "imageId", *runInstancesInput.ImageId, "err", err) - errResp = utils.GenerateErrorPayload(awserrors.ErrorInvalidAMIIDNotFound) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - } - - // Validate key pair exists (if specified) - if runInstancesInput.KeyName != nil && *runInstancesInput.KeyName != "" { - if err := d.keyService.ValidateKeyPairExists(*runInstancesInput.KeyName); err != nil { - slog.Error("handleEC2RunInstances key pair not found", "keyName", *runInstancesInput.KeyName, "err", err) - errResp = utils.GenerateErrorPayload(awserrors.ErrorInvalidKeyPairNotFound) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - } - - // Determine how many instances to launch based on MinCount/MaxCount - minCount := int(*runInstancesInput.MinCount) - maxCount := int(*runInstancesInput.MaxCount) - - // Check how many we can actually launch - allocatableCount := d.resourceMgr.canAllocate(instanceType, maxCount) - - if allocatableCount < minCount { - // Cannot satisfy MinCount requirement - fail entirely - slog.Error("handleEC2RunInstances insufficient capacity", "requested", minCount, "available", allocatableCount, "InstanceType", *runInstancesInput.InstanceType) - errResp = utils.GenerateErrorPayload(awserrors.ErrorInsufficientInstanceCapacity) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - // Launch up to MaxCount, capped by available capacity - // Note: canAllocate() already caps at maxCount, so allocatableCount <= maxCount - launchCount := allocatableCount - - slog.Info("Instance count determined", "min", minCount, "max", maxCount, "launching", launchCount) - - // Allocate resources for all instances upfront - var allocatedCount int - for i := 0; i < launchCount; i++ { - if err := d.resourceMgr.allocate(instanceType); err != nil { - slog.Error("handleEC2RunInstances allocate failed mid-allocation", "allocated", allocatedCount, "err", err) - break - } - allocatedCount++ - } - - // Check if we still meet MinCount after allocation - if allocatedCount < minCount { - // Rollback allocations - for i := 0; i < allocatedCount; i++ { - d.resourceMgr.deallocate(instanceType) - } - slog.Error("handleEC2RunInstances insufficient capacity after allocation", "allocated", allocatedCount, "minCount", minCount) - errResp = utils.GenerateErrorPayload(awserrors.ErrorInsufficientInstanceCapacity) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - // Update launchCount to what we actually allocated - launchCount = allocatedCount - - // Delegate to service for business logic (volume creation, cloud-init, etc.) - instanceTypeName := "" - if instanceType.InstanceType != nil { - instanceTypeName = *instanceType.InstanceType - } - slog.Info("Launching EC2 instances", "instanceType", instanceTypeName, "count", launchCount) - - // Create all instances - var instances []*vm.VM - var allEC2Instances []*ec2.Instance - var lastRunErr error - - for i := 0; i < launchCount; i++ { - instance, ec2Instance, err := d.instanceService.RunInstance(runInstancesInput) - if err != nil { - slog.Error("handleEC2RunInstances service.RunInstance failed", "index", i, "err", err) - lastRunErr = err - // Deallocate this instance's resources - d.resourceMgr.deallocate(instanceType) - continue - } - - // Auto-create ENI when SubnetId is provided (matches AWS behavior) - if runInstancesInput.SubnetId != nil && *runInstancesInput.SubnetId != "" && d.vpcService != nil { - eniOut, eniErr := d.vpcService.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ - SubnetId: runInstancesInput.SubnetId, - Description: aws.String("Primary network interface for " + instance.ID), - }) - if eniErr != nil { - slog.Error("handleEC2RunInstances auto-create ENI failed", "instanceId", instance.ID, "subnetId", *runInstancesInput.SubnetId, "err", eniErr) - lastRunErr = eniErr - d.resourceMgr.deallocate(instanceType) - continue - } - - eni := eniOut.NetworkInterface - instance.ENIId = *eni.NetworkInterfaceId - instance.ENIMac = *eni.MacAddress - ec2Instance.SetPrivateIpAddress(*eni.PrivateIpAddress) - ec2Instance.SetSubnetId(*runInstancesInput.SubnetId) - ec2Instance.SetVpcId(*eni.VpcId) - ec2Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{ - { - NetworkInterfaceId: eni.NetworkInterfaceId, - PrivateIpAddress: eni.PrivateIpAddress, - MacAddress: eni.MacAddress, - SubnetId: runInstancesInput.SubnetId, - VpcId: eni.VpcId, - Status: aws.String("in-use"), - Attachment: &ec2.InstanceNetworkInterfaceAttachment{ - DeviceIndex: aws.Int64(0), - Status: aws.String("attached"), - }, - }, - } - - slog.Info("Auto-created ENI for VPC instance", - "instanceId", instance.ID, - "eniId", instance.ENIId, - "privateIp", *eni.PrivateIpAddress, - "mac", instance.ENIMac, - ) - } - - instances = append(instances, instance) - allEC2Instances = append(allEC2Instances, ec2Instance) - } - - // Check if we still have enough instances after creation errors - if len(instances) < minCount { - // Rollback: deallocate resources for successfully created instances - // (failed instances already deallocated their resources above) - for range instances { - d.resourceMgr.deallocate(instanceType) - } - // Propagate the service-layer error if it's a known AWS error code - errCode := awserrors.ErrorServerInternal - if lastRunErr != nil { - if _, ok := awserrors.ErrorLookup[lastRunErr.Error()]; ok { - errCode = lastRunErr.Error() - } - } - slog.Error("handleEC2RunInstances failed to create minimum instances", "created", len(instances), "minCount", minCount, "err", errCode) - errResp = utils.GenerateErrorPayload(errCode) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - // Build reservation with all instances - reservation := ec2.Reservation{} - reservation.SetReservationId(utils.GenerateResourceID("r")) - reservation.SetOwnerId("123456789012") // TODO: Use actual owner ID from config - reservation.Instances = allEC2Instances - - // Store reservation reference in all VMs - for _, instance := range instances { - instance.Reservation = &reservation - } - - // Respond to NATS immediately with reservation (instances are provisioning) - jsonResponse, err := json.Marshal(reservation) - if err != nil { - slog.Error("handleEC2RunInstances failed to marshal reservation", "err", err) - errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - // Deallocate all resources - for range instances { - d.resourceMgr.deallocate(instanceType) - } - return - } - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - // Add all instances to state immediately so DescribeInstances can find them - // while volumes are being prepared and VMs are launching - d.Instances.Mu.Lock() - for _, instance := range instances { - d.Instances.VMS[instance.ID] = instance - } - d.Instances.Mu.Unlock() - - if err := d.WriteState(); err != nil { - slog.Error("handleEC2RunInstances failed to write initial state", "err", err) - } - - slog.Info("Instances added to state with pending status", "count", len(instances)) - - // Launch all instances (volumes and VMs) - var successCount int - for _, instance := range instances { - // Prepare the root volume, cloud-init, EFI drives via NBD (AMI clone to new volume) - volumeInfos, err := d.instanceService.GenerateVolumes(runInstancesInput, instance) - if err != nil { - slog.Error("handleEC2RunInstances GenerateVolumes failed", "instanceId", instance.ID, "err", err) - d.resourceMgr.deallocate(instanceType) - d.markInstanceFailed(instance, "volume_preparation_failed") - continue - } - - // Populate BlockDeviceMappings on the ec2.Instance - instance.Instance.BlockDeviceMappings = make([]*ec2.InstanceBlockDeviceMapping, 0, len(volumeInfos)) - for _, vi := range volumeInfos { - mapping := &ec2.InstanceBlockDeviceMapping{} - mapping.SetDeviceName(vi.DeviceName) - mapping.Ebs = &ec2.EbsInstanceBlockDevice{} - mapping.Ebs.SetVolumeId(vi.VolumeId) - mapping.Ebs.SetAttachTime(vi.AttachTime) - mapping.Ebs.SetDeleteOnTermination(vi.DeleteOnTermination) - mapping.Ebs.SetStatus("attached") - instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping) - } - - // Launch the instance infrastructure (QEMU, QMP, NATS subscriptions) - err = d.LaunchInstance(instance) - if err != nil { - slog.Error("handleEC2RunInstances LaunchInstance failed", "instanceId", instance.ID, "err", err) - d.resourceMgr.deallocate(instanceType) - d.markInstanceFailed(instance, "launch_failed") - continue - } - - successCount++ - slog.Info("handleEC2RunInstances launched instance", "instanceId", instance.ID) - } - - slog.Info("handleEC2RunInstances completed", "requested", launchCount, "created", len(instances), "launched", successCount) -} - -func (d *Daemon) handleEC2CreateKeyPair(msg *nats.Msg) { - handleNATSRequest(msg, d.keyService.CreateKeyPair) -} - -func (d *Daemon) handleEC2DeleteKeyPair(msg *nats.Msg) { - handleNATSRequest(msg, d.keyService.DeleteKeyPair) -} - -func (d *Daemon) handleEC2DescribeKeyPairs(msg *nats.Msg) { - handleNATSRequest(msg, d.keyService.DescribeKeyPairs) -} - -func (d *Daemon) handleEC2ImportKeyPair(msg *nats.Msg) { - handleNATSRequest(msg, d.keyService.ImportKeyPair) -} - -func (d *Daemon) handleEC2DescribeImages(msg *nats.Msg) { - handleNATSRequest(msg, d.imageService.DescribeImages) -} - -// handleEC2CreateImage is a stateful handler that extracts instance context -// (root volume ID, source AMI, running state) before delegating to the image service. -func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) { - slog.Debug("Received message", "subject", msg.Subject) - - respondWithError := func(errCode string) { - if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - } - - input := &ec2.CreateImageInput{} - if errResp := utils.UnmarshalJsonPayload(input, msg.Data); errResp != nil { - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - if input.InstanceId == nil || *input.InstanceId == "" { - respondWithError(awserrors.ErrorMissingParameter) - return - } - - instanceID := *input.InstanceId - - // Extract all instance context in a single critical section - d.Instances.Mu.Lock() - instance, ok := d.Instances.VMS[instanceID] - var status vm.InstanceState - var rootVolumeID, sourceImageID string - if ok { - status = instance.Status - if instance.Instance != nil { - for _, bdm := range instance.Instance.BlockDeviceMappings { - if bdm.Ebs != nil && bdm.Ebs.VolumeId != nil { - rootVolumeID = *bdm.Ebs.VolumeId - break - } - } - if instance.Instance.ImageId != nil { - sourceImageID = *instance.Instance.ImageId - } - } - } - d.Instances.Mu.Unlock() - - if !ok { - slog.Warn("CreateImage: instance not found", "instanceId", instanceID) - respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) - return - } - - if status != vm.StateRunning && status != vm.StateStopped { - slog.Warn("CreateImage: instance not in valid state", "instanceId", instanceID, "status", status) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - if rootVolumeID == "" { - slog.Error("CreateImage: no root volume found", "instanceId", instanceID) - respondWithError(awserrors.ErrorServerInternal) - return - } - - params := handlers_ec2_image.CreateImageParams{ - Input: input, - RootVolumeID: rootVolumeID, - SourceImageID: sourceImageID, - IsRunning: status == vm.StateRunning, - } - - output, err := d.imageService.CreateImageFromInstance(params) - if err != nil { - slog.Error("CreateImage: service failed", "instanceId", instanceID, "err", err) - respondWithError(err.Error()) - return - } - - jsonResponse, err := json.Marshal(output) - if err != nil { - slog.Error("CreateImage: failed to marshal response", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - slog.Info("CreateImage completed", "instanceId", instanceID, "imageId", *output.ImageId) -} - -func (d *Daemon) handleEC2CreateVolume(msg *nats.Msg) { - handleNATSRequest(msg, d.volumeService.CreateVolume) -} - -func (d *Daemon) handleEC2DescribeVolumes(msg *nats.Msg) { - handleNATSRequest(msg, d.volumeService.DescribeVolumes) -} - -func (d *Daemon) handleEC2DescribeVolumeStatus(msg *nats.Msg) { - handleNATSRequest(msg, d.volumeService.DescribeVolumeStatus) -} - -// handleEC2ModifyVolume processes incoming EC2 ModifyVolume requests -func (d *Daemon) handleEC2ModifyVolume(msg *nats.Msg) { - slog.Debug("Received message", "subject", msg.Subject) - slog.Debug("Message data", "data", string(msg.Data)) - - modifyVolumeInput := &ec2.ModifyVolumeInput{} - var errResp []byte - - errResp = utils.UnmarshalJsonPayload(modifyVolumeInput, msg.Data) - - if errResp != nil { - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - slog.Error("Request does not match ModifyVolumeInput") - return - } - - slog.Info("Processing ModifyVolume request", "volumeId", modifyVolumeInput.VolumeId) - - output, err := d.volumeService.ModifyVolume(modifyVolumeInput) - - if err != nil { - slog.Error("handleEC2ModifyVolume service.ModifyVolume failed", "err", err) - errResp = utils.GenerateErrorPayload(err.Error()) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - jsonResponse, err := json.Marshal(output) - if err != nil { - slog.Error("handleEC2ModifyVolume failed to marshal output", "err", err) - errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - // Notify viperblockd to reload state after volume modification (e.g. resize) - if modifyVolumeInput.VolumeId != nil { - syncData, err := json.Marshal(config.EBSSyncRequest{Volume: *modifyVolumeInput.VolumeId}) - if err != nil { - slog.Error("failed to marshal ebs.sync request", "volumeId", *modifyVolumeInput.VolumeId, "err", err) - } else { - _, syncErr := d.natsConn.Request("ebs.sync", syncData, 5*time.Second) - if syncErr != nil { - slog.Warn("ebs.sync notification failed (volume may not be mounted)", - "volumeId", *modifyVolumeInput.VolumeId, "err", syncErr) - } - } - } - - slog.Info("handleEC2ModifyVolume completed", "volumeId", modifyVolumeInput.VolumeId) -} - -func (d *Daemon) handleEC2DeleteVolume(msg *nats.Msg) { - handleNATSRequest(msg, d.volumeService.DeleteVolume) -} - -func (d *Daemon) handleEC2CreateSnapshot(msg *nats.Msg) { - handleNATSRequest(msg, d.snapshotService.CreateSnapshot) -} - -func (d *Daemon) handleEC2DescribeSnapshots(msg *nats.Msg) { - handleNATSRequest(msg, d.snapshotService.DescribeSnapshots) -} - -func (d *Daemon) handleEC2DeleteSnapshot(msg *nats.Msg) { - handleNATSRequest(msg, d.snapshotService.DeleteSnapshot) -} - -func (d *Daemon) handleEC2CopySnapshot(msg *nats.Msg) { - handleNATSRequest(msg, d.snapshotService.CopySnapshot) -} - -func (d *Daemon) handleEC2CreateTags(msg *nats.Msg) { - handleNATSRequest(msg, d.tagsService.CreateTags) -} - -func (d *Daemon) handleEC2DeleteTags(msg *nats.Msg) { - handleNATSRequest(msg, d.tagsService.DeleteTags) -} - -func (d *Daemon) handleEC2DescribeTags(msg *nats.Msg) { - handleNATSRequest(msg, d.tagsService.DescribeTags) -} - -func (d *Daemon) handleEC2CreateEgressOnlyInternetGateway(msg *nats.Msg) { - handleNATSRequest(msg, d.eigwService.CreateEgressOnlyInternetGateway) -} - -func (d *Daemon) handleEC2DeleteEgressOnlyInternetGateway(msg *nats.Msg) { - handleNATSRequest(msg, d.eigwService.DeleteEgressOnlyInternetGateway) -} - -func (d *Daemon) handleEC2DescribeEgressOnlyInternetGateways(msg *nats.Msg) { - handleNATSRequest(msg, d.eigwService.DescribeEgressOnlyInternetGateways) -} - -func (d *Daemon) handleEC2CreateInternetGateway(msg *nats.Msg) { - handleNATSRequest(msg, d.igwService.CreateInternetGateway) -} - -func (d *Daemon) handleEC2DeleteInternetGateway(msg *nats.Msg) { - handleNATSRequest(msg, d.igwService.DeleteInternetGateway) -} - -func (d *Daemon) handleEC2DescribeInternetGateways(msg *nats.Msg) { - handleNATSRequest(msg, d.igwService.DescribeInternetGateways) -} - -func (d *Daemon) handleEC2AttachInternetGateway(msg *nats.Msg) { - handleNATSRequest(msg, d.igwService.AttachInternetGateway) -} - -func (d *Daemon) handleEC2DetachInternetGateway(msg *nats.Msg) { - handleNATSRequest(msg, d.igwService.DetachInternetGateway) -} - -func (d *Daemon) handleEC2CreateVpc(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.CreateVpc) -} - -func (d *Daemon) handleEC2DeleteVpc(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.DeleteVpc) -} - -func (d *Daemon) handleEC2DescribeVpcs(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.DescribeVpcs) -} - -func (d *Daemon) handleEC2CreateSubnet(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.CreateSubnet) -} - -func (d *Daemon) handleEC2DeleteSubnet(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.DeleteSubnet) -} - -func (d *Daemon) handleEC2DescribeSubnets(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.DescribeSubnets) -} - -func (d *Daemon) handleEC2CreateNetworkInterface(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.CreateNetworkInterface) -} - -func (d *Daemon) handleEC2DeleteNetworkInterface(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.DeleteNetworkInterface) -} - -func (d *Daemon) handleEC2DescribeNetworkInterfaces(msg *nats.Msg) { - handleNATSRequest(msg, d.vpcService.DescribeNetworkInterfaces) -} - -// handleEC2DescribeInstanceTypes processes incoming EC2 DescribeInstanceTypes requests -// This handler responds with instance types that can currently be provisioned on this node -// based on available resources (CPU and memory not already allocated to running instances) -func (d *Daemon) handleEC2DescribeInstanceTypes(msg *nats.Msg) { - slog.Debug("Received message", "subject", msg.Subject) - - // Initialize input - describeInput := &ec2.DescribeInstanceTypesInput{} - var errResp []byte - - errResp = utils.UnmarshalJsonPayload(describeInput, msg.Data) - if errResp != nil { - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - slog.Error("Request does not match DescribeInstanceTypesInput") - return - } - - slog.Info("Processing DescribeInstanceTypes request from this node") - - // Check if "capacity" filter is set to "true" - showCapacity := false - for _, f := range describeInput.Filters { - if f.Name != nil && *f.Name == "capacity" { - for _, v := range f.Values { - if v != nil && *v == "true" { - showCapacity = true - break - } - } - } - } - - // Get instance types based on capacity and the showCapacity flag - filteredTypes := d.resourceMgr.GetAvailableInstanceTypeInfos(showCapacity) - - // Create the response - output := &ec2.DescribeInstanceTypesOutput{ - InstanceTypes: filteredTypes, - } - - // Respond to NATS - jsonResponse, err := json.Marshal(output) - if err != nil { - slog.Error("handleEC2DescribeInstanceTypes failed to marshal output", "err", err) - errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - slog.Info("handleEC2DescribeInstanceTypes completed", "count", len(filteredTypes)) -} - -// handleEC2DescribeInstances processes incoming EC2 DescribeInstances requests -// This handler responds with all instances running on this node -func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) { - slog.Debug("Received message", "subject", msg.Subject, "data", string(msg.Data)) - - // Initialize describeInstancesInput before unmarshaling into it - describeInstancesInput := &ec2.DescribeInstancesInput{} - var errResp []byte - - errResp = utils.UnmarshalJsonPayload(describeInstancesInput, msg.Data) - - if errResp != nil { - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - slog.Error("Request does not match DescribeInstancesInput") - return - } - - slog.Info("Processing DescribeInstances request from this node") - - d.Instances.Mu.Lock() - defer d.Instances.Mu.Unlock() - - // Validate and filter instances if specific instance IDs were requested - instanceIDFilter := make(map[string]bool) - if len(describeInstancesInput.InstanceIds) > 0 { - for _, id := range describeInstancesInput.InstanceIds { - if id != nil && *id != "" { - if !strings.HasPrefix(*id, "i-") { - if err := msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDMalformed)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - instanceIDFilter[*id] = true - } - } - } - - // Group instances by reservation ID (AWS returns instances grouped by reservation) - reservationMap := make(map[string]*ec2.Reservation) - - // Iterate through all instances on this node - for _, instance := range d.Instances.VMS { - // Skip if filtering by instance IDs and this instance is not in the filter - if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] { - continue - } - - // Use stored reservation metadata if available - if instance.Reservation != nil && instance.Instance != nil { - resID := "" - if instance.Reservation.ReservationId != nil { - resID = *instance.Reservation.ReservationId - } - - // Create reservation entry if it doesn't exist - if _, exists := reservationMap[resID]; !exists { - reservation := &ec2.Reservation{} - reservation.SetReservationId(resID) - if instance.Reservation.OwnerId != nil { - reservation.SetOwnerId(*instance.Reservation.OwnerId) - } - reservation.Instances = []*ec2.Instance{} - reservationMap[resID] = reservation - } - - // Update the instance state to current state - instanceCopy := *instance.Instance - instanceCopy.State = &ec2.InstanceState{} - - // Map internal status to EC2 state codes using the centralized mapping - if info, ok := vm.EC2StateCodes[instance.Status]; ok { - instanceCopy.State.SetCode(info.Code) - instanceCopy.State.SetName(info.Name) - } else { - slog.Warn("Instance has unmapped status, reporting as pending", - "instanceId", instance.ID, "status", string(instance.Status)) - instanceCopy.State.SetCode(0) - instanceCopy.State.SetName("pending") - } - - // Add instance to its reservation - reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy) - } - } - - // Convert map to slice - reservations := make([]*ec2.Reservation, 0, len(reservationMap)) - for _, reservation := range reservationMap { - reservations = append(reservations, reservation) - } - - // Create the response - output := &ec2.DescribeInstancesOutput{ - Reservations: reservations, - } - - // Respond to NATS with DescribeInstancesOutput - jsonResponse, err := json.Marshal(output) - if err != nil { - slog.Error("handleEC2DescribeInstances failed to marshal output", "err", err) - errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - slog.Info("handleEC2DescribeInstances completed", "count", len(reservations)) -} - -// handleEC2GetConsoleOutput reads the console log file for an instance and returns -// base64-encoded output matching the AWS GetConsoleOutput API response format. -func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) { - slog.Debug("Received GetConsoleOutput request", "subject", msg.Subject, "data", string(msg.Data)) - - var input ec2.GetConsoleOutputInput - if errResp := utils.UnmarshalJsonPayload(&input, msg.Data); errResp != nil { - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - if input.InstanceId == nil { - errResp := utils.GenerateErrorPayload(awserrors.ErrorMissingParameter) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - // Check account-level serial console access - status, err := d.accountService.GetSerialConsoleAccessStatus(&ec2.GetSerialConsoleAccessStatusInput{}) - if err != nil { - slog.Error("Failed to check serial console access status", "err", err) - if err := msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorServerInternal)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - if status.SerialConsoleAccessEnabled == nil || !*status.SerialConsoleAccessEnabled { - if err := msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorSerialConsoleSessionUnavailable)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - instanceID := *input.InstanceId - - // Find the instance on this node - d.Instances.Mu.Lock() - instance, exists := d.Instances.VMS[instanceID] - d.Instances.Mu.Unlock() - - if !exists { - errResp := utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - logPath := instance.Config.ConsoleLogPath - var outputData []byte - var modTime time.Time - - if logPath != "" { - info, err := os.Stat(logPath) - if err == nil { - modTime = info.ModTime() - - data, err := os.ReadFile(logPath) - if err != nil { - slog.Error("Failed to read console log", "path", logPath, "err", err) - } else { - // Return last 64KB (AWS limit) - const maxConsoleOutput = 64 * 1024 - if len(data) > maxConsoleOutput { - data = data[len(data)-maxConsoleOutput:] - } - outputData = data - } - } - } - - // Base64-encode the output (AWS returns base64) - var encodedOutput string - if len(outputData) > 0 { - encodedOutput = base64.StdEncoding.EncodeToString(outputData) - } - - now := time.Now() - if modTime.IsZero() { - modTime = now - } - - output := &ec2.GetConsoleOutputOutput{ - InstanceId: &instanceID, - Output: &encodedOutput, - Timestamp: &modTime, - } - - jsonResponse, err := json.Marshal(output) - if err != nil { - slog.Error("Failed to marshal GetConsoleOutput response", "err", err) - errResp := utils.GenerateErrorPayload(awserrors.ErrorServerInternal) - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - slog.Info("handleEC2GetConsoleOutput completed", "instance_id", instanceID, "output_bytes", len(outputData)) -} +// --- Admin / node management handlers --- // handleHealthCheck processes NATS health check requests func (d *Daemon) handleHealthCheck(msg *nats.Msg) { @@ -1603,345 +181,6 @@ func (d *Daemon) handleNodeDiscover(msg *nats.Msg) { slog.Debug("Node discovery responded", "node", d.node) } -// Account settings handlers - -func (d *Daemon) handleEC2EnableEbsEncryptionByDefault(msg *nats.Msg) { - handleNATSRequest(msg, d.accountService.EnableEbsEncryptionByDefault) -} - -func (d *Daemon) handleEC2DisableEbsEncryptionByDefault(msg *nats.Msg) { - handleNATSRequest(msg, d.accountService.DisableEbsEncryptionByDefault) -} - -func (d *Daemon) handleEC2GetEbsEncryptionByDefault(msg *nats.Msg) { - handleNATSRequest(msg, d.accountService.GetEbsEncryptionByDefault) -} - -func (d *Daemon) handleEC2GetSerialConsoleAccessStatus(msg *nats.Msg) { - handleNATSRequest(msg, d.accountService.GetSerialConsoleAccessStatus) -} - -func (d *Daemon) handleEC2EnableSerialConsoleAccess(msg *nats.Msg) { - handleNATSRequest(msg, d.accountService.EnableSerialConsoleAccess) -} - -func (d *Daemon) handleEC2DisableSerialConsoleAccess(msg *nats.Msg) { - handleNATSRequest(msg, d.accountService.DisableSerialConsoleAccess) -} - -// startStoppedInstanceRequest is the payload for ec2.start topic -type startStoppedInstanceRequest struct { - InstanceID string `json:"instance_id"` -} - -// handleEC2StartStoppedInstance picks up a stopped instance from shared KV, -// re-launches it on this daemon node, and removes it from shared KV. -func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) { - respondWithError := func(errCode string) { - if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - } - - var req startStoppedInstanceRequest - if err := json.Unmarshal(msg.Data, &req); err != nil { - slog.Error("handleEC2StartStoppedInstance: failed to unmarshal request", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - if req.InstanceID == "" { - slog.Error("handleEC2StartStoppedInstance: missing instance_id") - respondWithError(awserrors.ErrorMissingParameter) - return - } - - if d.jsManager == nil { - slog.Error("handleEC2StartStoppedInstance: JetStream not available") - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Load instance from shared KV - instance, err := d.jsManager.LoadStoppedInstance(req.InstanceID) - if err != nil { - slog.Error("handleEC2StartStoppedInstance: failed to load stopped instance", "instanceId", req.InstanceID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - if instance == nil { - slog.Warn("handleEC2StartStoppedInstance: instance not found in shared KV", "instanceId", req.InstanceID) - respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) - return - } - - if instance.Status != vm.StateStopped { - slog.Error("handleEC2StartStoppedInstance: instance not in stopped state", "instanceId", req.InstanceID, "status", instance.Status) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - // Reset node-local fields that are stale after cross-node migration - instance.ResetNodeLocalState() - - // Allocate resources - instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType] - if !ok { - slog.Error("handleEC2StartStoppedInstance: instance type not available on this node", - "instanceId", req.InstanceID, "instanceType", instance.InstanceType) - respondWithError(awserrors.ErrorInsufficientInstanceCapacity) - return - } - if err := d.resourceMgr.allocate(instanceType); err != nil { - slog.Error("handleEC2StartStoppedInstance: failed to allocate resources", "instanceId", req.InstanceID, "err", err) - respondWithError(awserrors.ErrorInsufficientInstanceCapacity) - return - } - - // Add instance to local map and clear stop attribute before launch - d.Instances.Mu.Lock() - d.Instances.VMS[instance.ID] = instance - instance.Attributes = qmp.Attributes{StartInstance: true} - d.Instances.Mu.Unlock() - - // Launch the instance infrastructure (QEMU, QMP, NATS subscriptions) - err = d.LaunchInstance(instance) - if err != nil { - slog.Error("handleEC2StartStoppedInstance: LaunchInstance failed", "instanceId", req.InstanceID, "err", err) - // Rollback: deallocate resources and remove from local map - if ok { - d.resourceMgr.deallocate(instanceType) - } - d.Instances.Mu.Lock() - delete(d.Instances.VMS, instance.ID) - d.Instances.Mu.Unlock() - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Remove from shared KV now that it's running locally. - // Retry once on failure — a stale KV entry risks duplicate starts. - if err := d.jsManager.DeleteStoppedInstance(req.InstanceID); err != nil { - slog.Warn("handleEC2StartStoppedInstance: first KV delete failed, retrying", - "instanceId", req.InstanceID, "err", err) - if retryErr := d.jsManager.DeleteStoppedInstance(req.InstanceID); retryErr != nil { - slog.Error("handleEC2StartStoppedInstance: KV delete failed after retry, instance is running locally but stale entry remains in shared KV", - "instanceId", req.InstanceID, "err", retryErr) - } - } - - slog.Info("Started stopped instance from shared KV", "instanceId", instance.ID, "node", d.node) - - if err := msg.Respond(fmt.Appendf(nil, `{"status":"running","instanceId":"%s"}`, instance.ID)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } -} - -// terminateStoppedInstanceRequest is the payload for ec2.terminate topic -type terminateStoppedInstanceRequest struct { - InstanceID string `json:"instance_id"` -} - -// handleEC2TerminateStoppedInstance picks up a stopped instance from shared KV, -// deletes its volumes, and removes it from shared KV. -func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) { - respondWithError := func(errCode string) { - if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - } - - var req terminateStoppedInstanceRequest - if err := json.Unmarshal(msg.Data, &req); err != nil { - slog.Error("handleEC2TerminateStoppedInstance: failed to unmarshal request", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - if req.InstanceID == "" { - slog.Error("handleEC2TerminateStoppedInstance: missing instance_id") - respondWithError(awserrors.ErrorMissingParameter) - return - } - - if d.jsManager == nil { - slog.Error("handleEC2TerminateStoppedInstance: JetStream not available") - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Load instance from shared KV - instance, err := d.jsManager.LoadStoppedInstance(req.InstanceID) - if err != nil { - slog.Error("handleEC2TerminateStoppedInstance: failed to load stopped instance", "instanceId", req.InstanceID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - if instance == nil { - slog.Warn("handleEC2TerminateStoppedInstance: instance not found in shared KV", "instanceId", req.InstanceID) - respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) - return - } - - if instance.Status != vm.StateStopped { - slog.Error("handleEC2TerminateStoppedInstance: instance not in stopped state", "instanceId", req.InstanceID, "status", instance.Status) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - // Delete volumes — no QEMU shutdown or unmount needed (already done during stop) - instance.EBSRequests.Mu.Lock() - for _, ebsRequest := range instance.EBSRequests.Requests { - // Internal volumes (EFI, cloud-init) are always cleaned up via ebs.delete - if ebsRequest.EFI || ebsRequest.CloudInit { - ebsDeleteData, err := json.Marshal(config.EBSDeleteRequest{Volume: ebsRequest.Name}) - if err != nil { - slog.Error("handleEC2TerminateStoppedInstance: failed to marshal ebs.delete request", "name", ebsRequest.Name, "err", err) - continue - } - deleteMsg, err := d.natsConn.Request("ebs.delete", ebsDeleteData, 30*time.Second) - if err != nil { - slog.Warn("handleEC2TerminateStoppedInstance: ebs.delete failed for internal volume", "name", ebsRequest.Name, "err", err) - } else { - slog.Info("handleEC2TerminateStoppedInstance: ebs.delete sent for internal volume", "name", ebsRequest.Name, "data", string(deleteMsg.Data)) - } - continue - } - - // User-visible volumes: respect DeleteOnTermination flag - if !ebsRequest.DeleteOnTermination { - slog.Info("handleEC2TerminateStoppedInstance: volume has DeleteOnTermination=false, skipping", "name", ebsRequest.Name) - continue - } - - slog.Info("handleEC2TerminateStoppedInstance: deleting volume with DeleteOnTermination=true", "name", ebsRequest.Name) - _, err := d.volumeService.DeleteVolume(&ec2.DeleteVolumeInput{ - VolumeId: &ebsRequest.Name, - }) - if err != nil { - slog.Error("handleEC2TerminateStoppedInstance: failed to delete volume", "name", ebsRequest.Name, "err", err) - } - } - instance.EBSRequests.Mu.Unlock() - - // Remove from shared KV - if err := d.jsManager.DeleteStoppedInstance(req.InstanceID); err != nil { - slog.Error("handleEC2TerminateStoppedInstance: failed to delete from shared KV", "instanceId", req.InstanceID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - slog.Info("Terminated stopped instance from shared KV", "instanceId", req.InstanceID) - - if err := msg.Respond(fmt.Appendf(nil, `{"status":"terminated","instanceId":"%s"}`, req.InstanceID)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } -} - -// handleEC2DescribeStoppedInstances returns stopped instances from shared KV. -func (d *Daemon) handleEC2DescribeStoppedInstances(msg *nats.Msg) { - respondWithError := func(errCode string) { - if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - } - - if d.jsManager == nil { - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Parse optional filters from the request - describeInput := &ec2.DescribeInstancesInput{} - if len(msg.Data) > 0 { - if errResp := utils.UnmarshalJsonPayload(describeInput, msg.Data); errResp != nil { - if err := msg.Respond(errResp); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - return - } - } - - // Build instance ID filter - instanceIDFilter := make(map[string]bool) - if len(describeInput.InstanceIds) > 0 { - for _, id := range describeInput.InstanceIds { - if id != nil { - instanceIDFilter[*id] = true - } - } - } - - instances, err := d.jsManager.ListStoppedInstances() - if err != nil { - slog.Error("handleEC2DescribeStoppedInstances: failed to list stopped instances", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - // Group instances by reservation ID - reservationMap := make(map[string]*ec2.Reservation) - - for _, instance := range instances { - // Apply instance ID filter - if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] { - continue - } - - if instance.Reservation != nil && instance.Instance != nil { - resID := "" - if instance.Reservation.ReservationId != nil { - resID = *instance.Reservation.ReservationId - } - - if _, exists := reservationMap[resID]; !exists { - reservation := &ec2.Reservation{} - reservation.SetReservationId(resID) - if instance.Reservation.OwnerId != nil { - reservation.SetOwnerId(*instance.Reservation.OwnerId) - } - reservation.Instances = []*ec2.Instance{} - reservationMap[resID] = reservation - } - - // Update the instance state - instanceCopy := *instance.Instance - instanceCopy.State = &ec2.InstanceState{} - if info, ok := vm.EC2StateCodes[instance.Status]; ok { - instanceCopy.State.SetCode(info.Code) - instanceCopy.State.SetName(info.Name) - } else { - instanceCopy.State.SetCode(80) - instanceCopy.State.SetName("stopped") - } - - reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy) - } - } - - reservations := make([]*ec2.Reservation, 0, len(reservationMap)) - for _, reservation := range reservationMap { - reservations = append(reservations, reservation) - } - - output := &ec2.DescribeInstancesOutput{ - Reservations: reservations, - } - - jsonResponse, err := json.Marshal(output) - if err != nil { - slog.Error("handleEC2DescribeStoppedInstances: failed to marshal output", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - if err := msg.Respond(jsonResponse); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - - slog.Info("handleEC2DescribeStoppedInstances completed", "count", len(reservations)) -} - // daemonIP extracts the IP portion from the daemon host (host:port format). func (d *Daemon) daemonIP() string { host := d.config.Daemon.Host @@ -2029,98 +268,3 @@ func (d *Daemon) handleNodeVMs(msg *nats.Msg) { slog.Error("Failed to respond to hive.node.vms", "err", err) } } - -// handleEC2ModifyInstanceAttribute modifies attributes of a stopped instance in shared KV. -// All supported attributes (InstanceType, UserData) require the instance to be stopped. -func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) { - respondWithError := func(errCode string) { - if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } - } - - var input ec2.ModifyInstanceAttributeInput - if err := json.Unmarshal(msg.Data, &input); err != nil { - slog.Error("handleEC2ModifyInstanceAttribute: failed to unmarshal request", "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - if input.InstanceId == nil || *input.InstanceId == "" { - slog.Error("handleEC2ModifyInstanceAttribute: missing instance_id") - respondWithError(awserrors.ErrorMissingParameter) - return - } - - instanceID := *input.InstanceId - - if d.jsManager == nil { - slog.Error("handleEC2ModifyInstanceAttribute: JetStream not available") - respondWithError(awserrors.ErrorServerInternal) - return - } - - instance, err := d.jsManager.LoadStoppedInstance(instanceID) - if err != nil { - slog.Error("handleEC2ModifyInstanceAttribute: failed to load stopped instance", "instanceId", instanceID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - if instance == nil { - slog.Warn("handleEC2ModifyInstanceAttribute: instance not found in shared KV", "instanceId", instanceID) - respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) - return - } - - if instance.Status != vm.StateStopped { - slog.Error("handleEC2ModifyInstanceAttribute: instance not in stopped state", "instanceId", instanceID, "status", instance.Status) - respondWithError(awserrors.ErrorIncorrectInstanceState) - return - } - - // Apply the requested attribute change - if input.InstanceType != nil && input.InstanceType.Value != nil { - newType := *input.InstanceType.Value - if newType == "" { - slog.Error("handleEC2ModifyInstanceAttribute: empty instance type value", "instanceId", instanceID) - respondWithError(awserrors.ErrorInvalidInstanceAttributeValue) - return - } - if instance.Instance == nil { - slog.Error("handleEC2ModifyInstanceAttribute: instance.Instance is nil, data integrity issue", "instanceId", instanceID) - respondWithError(awserrors.ErrorServerInternal) - return - } - slog.Info("handleEC2ModifyInstanceAttribute: changing instance type", - "instanceId", instanceID, "oldType", instance.InstanceType, "newType", newType) - - instance.InstanceType = newType - instance.Config.InstanceType = newType - instance.Instance.InstanceType = aws.String(newType) - // Clear StateReason — resolves capacity-unavailable state from instance-type-missing bug - instance.Instance.StateReason = nil - } - - if input.UserData != nil && input.UserData.Value != nil { - slog.Info("handleEC2ModifyInstanceAttribute: changing user data", "instanceId", instanceID) - - // Value arrives as decoded bytes (JSON unmarshal handles base64 → []byte automatically) - instance.UserData = string(input.UserData.Value) - if instance.RunInstancesInput != nil { - instance.RunInstancesInput.UserData = aws.String(base64.StdEncoding.EncodeToString(input.UserData.Value)) - } - } - - if err := d.jsManager.WriteStoppedInstance(instanceID, instance); err != nil { - slog.Error("handleEC2ModifyInstanceAttribute: failed to write modified instance to KV", - "instanceId", instanceID, "err", err) - respondWithError(awserrors.ErrorServerInternal) - return - } - - slog.Info("handleEC2ModifyInstanceAttribute: completed successfully", "instanceId", instanceID) - - if err := msg.Respond([]byte(`{}`)); err != nil { - slog.Error("Failed to respond to NATS request", "err", err) - } -} diff --git a/hive/daemon/daemon_handlers_account.go b/hive/daemon/daemon_handlers_account.go new file mode 100644 index 000000000..0b5bc4123 --- /dev/null +++ b/hive/daemon/daemon_handlers_account.go @@ -0,0 +1,29 @@ +package daemon + +import ( + "github.com/nats-io/nats.go" +) + +func (d *Daemon) handleEC2EnableEbsEncryptionByDefault(msg *nats.Msg) { + handleNATSRequest(msg, d.accountService.EnableEbsEncryptionByDefault) +} + +func (d *Daemon) handleEC2DisableEbsEncryptionByDefault(msg *nats.Msg) { + handleNATSRequest(msg, d.accountService.DisableEbsEncryptionByDefault) +} + +func (d *Daemon) handleEC2GetEbsEncryptionByDefault(msg *nats.Msg) { + handleNATSRequest(msg, d.accountService.GetEbsEncryptionByDefault) +} + +func (d *Daemon) handleEC2GetSerialConsoleAccessStatus(msg *nats.Msg) { + handleNATSRequest(msg, d.accountService.GetSerialConsoleAccessStatus) +} + +func (d *Daemon) handleEC2EnableSerialConsoleAccess(msg *nats.Msg) { + handleNATSRequest(msg, d.accountService.EnableSerialConsoleAccess) +} + +func (d *Daemon) handleEC2DisableSerialConsoleAccess(msg *nats.Msg) { + handleNATSRequest(msg, d.accountService.DisableSerialConsoleAccess) +} diff --git a/hive/daemon/daemon_handlers_console.go b/hive/daemon/daemon_handlers_console.go new file mode 100644 index 000000000..9efeb58c9 --- /dev/null +++ b/hive/daemon/daemon_handlers_console.go @@ -0,0 +1,123 @@ +package daemon + +import ( + "encoding/base64" + "encoding/json" + "log/slog" + "os" + "time" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/hive/hive/awserrors" + "github.com/mulgadc/hive/hive/utils" + "github.com/nats-io/nats.go" +) + +// handleEC2GetConsoleOutput reads the console log file for an instance and returns +// base64-encoded output matching the AWS GetConsoleOutput API response format. +func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) { + slog.Debug("Received GetConsoleOutput request", "subject", msg.Subject, "data", string(msg.Data)) + + var input ec2.GetConsoleOutputInput + if errResp := utils.UnmarshalJsonPayload(&input, msg.Data); errResp != nil { + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + if input.InstanceId == nil { + errResp := utils.GenerateErrorPayload(awserrors.ErrorMissingParameter) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + // Check account-level serial console access + status, err := d.accountService.GetSerialConsoleAccessStatus(&ec2.GetSerialConsoleAccessStatusInput{}) + if err != nil { + slog.Error("Failed to check serial console access status", "err", err) + if err := msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorServerInternal)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + if status.SerialConsoleAccessEnabled == nil || !*status.SerialConsoleAccessEnabled { + if err := msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorSerialConsoleSessionUnavailable)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + instanceID := *input.InstanceId + + // Find the instance on this node + d.Instances.Mu.Lock() + instance, exists := d.Instances.VMS[instanceID] + d.Instances.Mu.Unlock() + + if !exists { + errResp := utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + logPath := instance.Config.ConsoleLogPath + var outputData []byte + var modTime time.Time + + if logPath != "" { + info, err := os.Stat(logPath) + if err == nil { + modTime = info.ModTime() + + data, err := os.ReadFile(logPath) + if err != nil { + slog.Error("Failed to read console log", "path", logPath, "err", err) + } else { + // Return last 64KB (AWS limit) + const maxConsoleOutput = 64 * 1024 + if len(data) > maxConsoleOutput { + data = data[len(data)-maxConsoleOutput:] + } + outputData = data + } + } + } + + // Base64-encode the output (AWS returns base64) + var encodedOutput string + if len(outputData) > 0 { + encodedOutput = base64.StdEncoding.EncodeToString(outputData) + } + + now := time.Now() + if modTime.IsZero() { + modTime = now + } + + output := &ec2.GetConsoleOutputOutput{ + InstanceId: &instanceID, + Output: &encodedOutput, + Timestamp: &modTime, + } + + jsonResponse, err := json.Marshal(output) + if err != nil { + slog.Error("Failed to marshal GetConsoleOutput response", "err", err) + errResp := utils.GenerateErrorPayload(awserrors.ErrorServerInternal) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + if err := msg.Respond(jsonResponse); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + slog.Info("handleEC2GetConsoleOutput completed", "instance_id", instanceID, "output_bytes", len(outputData)) +} diff --git a/hive/daemon/daemon_handlers_igw.go b/hive/daemon/daemon_handlers_igw.go new file mode 100644 index 000000000..0e5b0b014 --- /dev/null +++ b/hive/daemon/daemon_handlers_igw.go @@ -0,0 +1,37 @@ +package daemon + +import ( + "github.com/nats-io/nats.go" +) + +func (d *Daemon) handleEC2CreateInternetGateway(msg *nats.Msg) { + handleNATSRequest(msg, d.igwService.CreateInternetGateway) +} + +func (d *Daemon) handleEC2DeleteInternetGateway(msg *nats.Msg) { + handleNATSRequest(msg, d.igwService.DeleteInternetGateway) +} + +func (d *Daemon) handleEC2DescribeInternetGateways(msg *nats.Msg) { + handleNATSRequest(msg, d.igwService.DescribeInternetGateways) +} + +func (d *Daemon) handleEC2AttachInternetGateway(msg *nats.Msg) { + handleNATSRequest(msg, d.igwService.AttachInternetGateway) +} + +func (d *Daemon) handleEC2DetachInternetGateway(msg *nats.Msg) { + handleNATSRequest(msg, d.igwService.DetachInternetGateway) +} + +func (d *Daemon) handleEC2CreateEgressOnlyInternetGateway(msg *nats.Msg) { + handleNATSRequest(msg, d.eigwService.CreateEgressOnlyInternetGateway) +} + +func (d *Daemon) handleEC2DeleteEgressOnlyInternetGateway(msg *nats.Msg) { + handleNATSRequest(msg, d.eigwService.DeleteEgressOnlyInternetGateway) +} + +func (d *Daemon) handleEC2DescribeEgressOnlyInternetGateways(msg *nats.Msg) { + handleNATSRequest(msg, d.eigwService.DescribeEgressOnlyInternetGateways) +} diff --git a/hive/daemon/daemon_handlers_image.go b/hive/daemon/daemon_handlers_image.go new file mode 100644 index 000000000..78665c1f0 --- /dev/null +++ b/hive/daemon/daemon_handlers_image.go @@ -0,0 +1,110 @@ +package daemon + +import ( + "encoding/json" + "log/slog" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/hive/hive/awserrors" + handlers_ec2_image "github.com/mulgadc/hive/hive/handlers/ec2/image" + "github.com/mulgadc/hive/hive/utils" + "github.com/mulgadc/hive/hive/vm" + "github.com/nats-io/nats.go" +) + +func (d *Daemon) handleEC2DescribeImages(msg *nats.Msg) { + handleNATSRequest(msg, d.imageService.DescribeImages) +} + +// handleEC2CreateImage is a stateful handler that extracts instance context +// (root volume ID, source AMI, running state) before delegating to the image service. +func (d *Daemon) handleEC2CreateImage(msg *nats.Msg) { + slog.Debug("Received message", "subject", msg.Subject) + + respondWithError := func(errCode string) { + if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + } + + input := &ec2.CreateImageInput{} + if errResp := utils.UnmarshalJsonPayload(input, msg.Data); errResp != nil { + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + if input.InstanceId == nil || *input.InstanceId == "" { + respondWithError(awserrors.ErrorMissingParameter) + return + } + + instanceID := *input.InstanceId + + // Extract all instance context in a single critical section + d.Instances.Mu.Lock() + instance, ok := d.Instances.VMS[instanceID] + var status vm.InstanceState + var rootVolumeID, sourceImageID string + if ok { + status = instance.Status + if instance.Instance != nil { + for _, bdm := range instance.Instance.BlockDeviceMappings { + if bdm.Ebs != nil && bdm.Ebs.VolumeId != nil { + rootVolumeID = *bdm.Ebs.VolumeId + break + } + } + if instance.Instance.ImageId != nil { + sourceImageID = *instance.Instance.ImageId + } + } + } + d.Instances.Mu.Unlock() + + if !ok { + slog.Warn("CreateImage: instance not found", "instanceId", instanceID) + respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) + return + } + + if status != vm.StateRunning && status != vm.StateStopped { + slog.Warn("CreateImage: instance not in valid state", "instanceId", instanceID, "status", status) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + if rootVolumeID == "" { + slog.Error("CreateImage: no root volume found", "instanceId", instanceID) + respondWithError(awserrors.ErrorServerInternal) + return + } + + params := handlers_ec2_image.CreateImageParams{ + Input: input, + RootVolumeID: rootVolumeID, + SourceImageID: sourceImageID, + IsRunning: status == vm.StateRunning, + } + + output, err := d.imageService.CreateImageFromInstance(params) + if err != nil { + slog.Error("CreateImage: service failed", "instanceId", instanceID, "err", err) + respondWithError(err.Error()) + return + } + + jsonResponse, err := json.Marshal(output) + if err != nil { + slog.Error("CreateImage: failed to marshal response", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + if err := msg.Respond(jsonResponse); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + slog.Info("CreateImage completed", "instanceId", instanceID, "imageId", *output.ImageId) +} diff --git a/hive/daemon/daemon_handlers_instance.go b/hive/daemon/daemon_handlers_instance.go new file mode 100644 index 000000000..6d7da1183 --- /dev/null +++ b/hive/daemon/daemon_handlers_instance.go @@ -0,0 +1,1036 @@ +package daemon + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/hive/hive/awserrors" + "github.com/mulgadc/hive/hive/config" + "github.com/mulgadc/hive/hive/qmp" + "github.com/mulgadc/hive/hive/utils" + "github.com/mulgadc/hive/hive/vm" + "github.com/nats-io/nats.go" +) + +// handleEC2RunInstances processes incoming EC2 RunInstances requests +func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) { + slog.Debug("Received message on subject", "subject", msg.Subject) + slog.Debug("Message data", "data", string(msg.Data)) + + // Initialize runInstancesInput before unmarshaling into it + runInstancesInput := &ec2.RunInstancesInput{} + var errResp []byte + + errResp = utils.UnmarshalJsonPayload(runInstancesInput, msg.Data) + + if errResp != nil { + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + slog.Error("Request does not match RunInstancesInput") + return + } + + slog.Info("Processing RunInstances request for instance type", "instanceType", *runInstancesInput.InstanceType) + + // Check if instance type is supported + instanceType, exists := d.resourceMgr.instanceTypes[*runInstancesInput.InstanceType] + if !exists { + slog.Error("handleEC2RunInstances instance lookup", "err", awserrors.ErrorInvalidInstanceType, "InstanceType", *runInstancesInput.InstanceType) + errResp = utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceType) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + // Validate AMI exists before allocating resources + if runInstancesInput.ImageId != nil { + _, err := d.imageService.GetAMIConfig(*runInstancesInput.ImageId) + if err != nil { + slog.Error("handleEC2RunInstances AMI not found", "imageId", *runInstancesInput.ImageId, "err", err) + errResp = utils.GenerateErrorPayload(awserrors.ErrorInvalidAMIIDNotFound) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + } + + // Validate key pair exists (if specified) + if runInstancesInput.KeyName != nil && *runInstancesInput.KeyName != "" { + if err := d.keyService.ValidateKeyPairExists(*runInstancesInput.KeyName); err != nil { + slog.Error("handleEC2RunInstances key pair not found", "keyName", *runInstancesInput.KeyName, "err", err) + errResp = utils.GenerateErrorPayload(awserrors.ErrorInvalidKeyPairNotFound) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + } + + // Determine how many instances to launch based on MinCount/MaxCount + minCount := int(*runInstancesInput.MinCount) + maxCount := int(*runInstancesInput.MaxCount) + + // Check how many we can actually launch + allocatableCount := d.resourceMgr.canAllocate(instanceType, maxCount) + + if allocatableCount < minCount { + // Cannot satisfy MinCount requirement - fail entirely + slog.Error("handleEC2RunInstances insufficient capacity", "requested", minCount, "available", allocatableCount, "InstanceType", *runInstancesInput.InstanceType) + errResp = utils.GenerateErrorPayload(awserrors.ErrorInsufficientInstanceCapacity) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + // Launch up to MaxCount, capped by available capacity + // Note: canAllocate() already caps at maxCount, so allocatableCount <= maxCount + launchCount := allocatableCount + + slog.Info("Instance count determined", "min", minCount, "max", maxCount, "launching", launchCount) + + // Allocate resources for all instances upfront + var allocatedCount int + for i := 0; i < launchCount; i++ { + if err := d.resourceMgr.allocate(instanceType); err != nil { + slog.Error("handleEC2RunInstances allocate failed mid-allocation", "allocated", allocatedCount, "err", err) + break + } + allocatedCount++ + } + + // Check if we still meet MinCount after allocation + if allocatedCount < minCount { + // Rollback allocations + for i := 0; i < allocatedCount; i++ { + d.resourceMgr.deallocate(instanceType) + } + slog.Error("handleEC2RunInstances insufficient capacity after allocation", "allocated", allocatedCount, "minCount", minCount) + errResp = utils.GenerateErrorPayload(awserrors.ErrorInsufficientInstanceCapacity) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + // Update launchCount to what we actually allocated + launchCount = allocatedCount + + // Delegate to service for business logic (volume creation, cloud-init, etc.) + instanceTypeName := "" + if instanceType.InstanceType != nil { + instanceTypeName = *instanceType.InstanceType + } + slog.Info("Launching EC2 instances", "instanceType", instanceTypeName, "count", launchCount) + + // Create all instances + var instances []*vm.VM + var allEC2Instances []*ec2.Instance + var lastRunErr error + + for i := 0; i < launchCount; i++ { + instance, ec2Instance, err := d.instanceService.RunInstance(runInstancesInput) + if err != nil { + slog.Error("handleEC2RunInstances service.RunInstance failed", "index", i, "err", err) + lastRunErr = err + // Deallocate this instance's resources + d.resourceMgr.deallocate(instanceType) + continue + } + + // Auto-create ENI when SubnetId is provided (matches AWS behavior) + if runInstancesInput.SubnetId != nil && *runInstancesInput.SubnetId != "" && d.vpcService != nil { + eniOut, eniErr := d.vpcService.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ + SubnetId: runInstancesInput.SubnetId, + Description: aws.String("Primary network interface for " + instance.ID), + }) + if eniErr != nil { + slog.Error("handleEC2RunInstances auto-create ENI failed", "instanceId", instance.ID, "subnetId", *runInstancesInput.SubnetId, "err", eniErr) + lastRunErr = eniErr + d.resourceMgr.deallocate(instanceType) + continue + } + + eni := eniOut.NetworkInterface + instance.ENIId = *eni.NetworkInterfaceId + instance.ENIMac = *eni.MacAddress + ec2Instance.SetPrivateIpAddress(*eni.PrivateIpAddress) + ec2Instance.SetSubnetId(*runInstancesInput.SubnetId) + ec2Instance.SetVpcId(*eni.VpcId) + ec2Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{ + { + NetworkInterfaceId: eni.NetworkInterfaceId, + PrivateIpAddress: eni.PrivateIpAddress, + MacAddress: eni.MacAddress, + SubnetId: runInstancesInput.SubnetId, + VpcId: eni.VpcId, + Status: aws.String("in-use"), + Attachment: &ec2.InstanceNetworkInterfaceAttachment{ + DeviceIndex: aws.Int64(0), + Status: aws.String("attached"), + }, + }, + } + + slog.Info("Auto-created ENI for VPC instance", + "instanceId", instance.ID, + "eniId", instance.ENIId, + "privateIp", *eni.PrivateIpAddress, + "mac", instance.ENIMac, + ) + } + + instances = append(instances, instance) + allEC2Instances = append(allEC2Instances, ec2Instance) + } + + // Check if we still have enough instances after creation errors + if len(instances) < minCount { + // Rollback: deallocate resources for successfully created instances + // (failed instances already deallocated their resources above) + for range instances { + d.resourceMgr.deallocate(instanceType) + } + // Propagate the service-layer error if it's a known AWS error code + errCode := awserrors.ErrorServerInternal + if lastRunErr != nil { + if _, ok := awserrors.ErrorLookup[lastRunErr.Error()]; ok { + errCode = lastRunErr.Error() + } + } + slog.Error("handleEC2RunInstances failed to create minimum instances", "created", len(instances), "minCount", minCount, "err", errCode) + errResp = utils.GenerateErrorPayload(errCode) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + // Build reservation with all instances + reservation := ec2.Reservation{} + reservation.SetReservationId(utils.GenerateResourceID("r")) + reservation.SetOwnerId("123456789012") // TODO: Use actual owner ID from config + reservation.Instances = allEC2Instances + + // Store reservation reference in all VMs + for _, instance := range instances { + instance.Reservation = &reservation + } + + // Respond to NATS immediately with reservation (instances are provisioning) + jsonResponse, err := json.Marshal(reservation) + if err != nil { + slog.Error("handleEC2RunInstances failed to marshal reservation", "err", err) + errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + // Deallocate all resources + for range instances { + d.resourceMgr.deallocate(instanceType) + } + return + } + if err := msg.Respond(jsonResponse); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + // Add all instances to state immediately so DescribeInstances can find them + // while volumes are being prepared and VMs are launching + d.Instances.Mu.Lock() + for _, instance := range instances { + d.Instances.VMS[instance.ID] = instance + } + d.Instances.Mu.Unlock() + + if err := d.WriteState(); err != nil { + slog.Error("handleEC2RunInstances failed to write initial state", "err", err) + } + + slog.Info("Instances added to state with pending status", "count", len(instances)) + + // Launch all instances (volumes and VMs) + var successCount int + for _, instance := range instances { + // Prepare the root volume, cloud-init, EFI drives via NBD (AMI clone to new volume) + volumeInfos, err := d.instanceService.GenerateVolumes(runInstancesInput, instance) + if err != nil { + slog.Error("handleEC2RunInstances GenerateVolumes failed", "instanceId", instance.ID, "err", err) + d.resourceMgr.deallocate(instanceType) + d.markInstanceFailed(instance, "volume_preparation_failed") + continue + } + + // Populate BlockDeviceMappings on the ec2.Instance + instance.Instance.BlockDeviceMappings = make([]*ec2.InstanceBlockDeviceMapping, 0, len(volumeInfos)) + for _, vi := range volumeInfos { + mapping := &ec2.InstanceBlockDeviceMapping{} + mapping.SetDeviceName(vi.DeviceName) + mapping.Ebs = &ec2.EbsInstanceBlockDevice{} + mapping.Ebs.SetVolumeId(vi.VolumeId) + mapping.Ebs.SetAttachTime(vi.AttachTime) + mapping.Ebs.SetDeleteOnTermination(vi.DeleteOnTermination) + mapping.Ebs.SetStatus("attached") + instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping) + } + + // Launch the instance infrastructure (QEMU, QMP, NATS subscriptions) + err = d.LaunchInstance(instance) + if err != nil { + slog.Error("handleEC2RunInstances LaunchInstance failed", "instanceId", instance.ID, "err", err) + d.resourceMgr.deallocate(instanceType) + d.markInstanceFailed(instance, "launch_failed") + continue + } + + successCount++ + slog.Info("handleEC2RunInstances launched instance", "instanceId", instance.ID) + } + + slog.Info("handleEC2RunInstances completed", "requested", launchCount, "created", len(instances), "launched", successCount) +} + +func (d *Daemon) handleStartInstance(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { + slog.Info("Starting instance", "id", command.ID) + + // Validate instance is in stopped state + d.Instances.Mu.Lock() + status := instance.Status + d.Instances.Mu.Unlock() + + if status != vm.StateStopped { + slog.Error("StartInstance: instance not in stopped state", "instanceId", command.ID, "status", status) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + // Allocate resources + instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType] + if ok { + if err := d.resourceMgr.allocate(instanceType); err != nil { + slog.Error("Failed to allocate resources for start command", "id", command.ID, "err", err) + respondWithError(awserrors.ErrorInsufficientInstanceCapacity) + return + } + } + + // Clear stop attribute before launch so WriteState inside LaunchInstance + // persists the correct attributes. Without this, a daemon restart after + // a stop→start cycle would see StopInstance=true and skip reconnecting QEMU. + d.Instances.Mu.Lock() + instance.Attributes = command.Attributes + d.Instances.Mu.Unlock() + + // Launch the instance infrastructure (QEMU, QMP, NATS subscriptions) + err := d.LaunchInstance(instance) + + if err != nil { + slog.Error("handleEC2RunInstances LaunchInstance failed", "err", err) + // Free the resource on failure + if ok { + d.resourceMgr.deallocate(instanceType) + } + respondWithError(awserrors.ErrorServerInternal) + return + } + + slog.Info("Instance started", "instanceId", instance.ID) + + if err := msg.Respond(fmt.Appendf(nil, `{"status":"running","instanceId":"%s"}`, instance.ID)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } +} + +func (d *Daemon) handleStopOrTerminateInstance(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { + isTerminate := command.Attributes.TerminateInstance + action := "Stopping" + initialState := vm.StateStopping + finalState := vm.StateStopped + if isTerminate { + action = "Terminating" + initialState = vm.StateShuttingDown + finalState = vm.StateTerminated + } + + slog.Info(action+" instance", "id", command.ID) + + // Check state validity before attempting transition — return the correct + // AWS error code when the instance is already stopped/terminated/etc. + d.Instances.Mu.Lock() + currentState := instance.Status + d.Instances.Mu.Unlock() + if !vm.IsValidTransition(currentState, initialState) { + slog.Warn("Instance in incorrect state for "+strings.ToLower(action), + "instanceId", instance.ID, "currentState", string(currentState)) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + // Transition to the initial transitional state + if err := d.TransitionState(instance, initialState); err != nil { + slog.Error("Failed to transition to "+string(initialState), "instanceId", instance.ID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Respond immediately - operation will complete in background + // stopInstance() handles the QMP shutdown command, so we don't send it here + if err := msg.Respond([]byte(`{}`)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + // Run cleanup in goroutine to not block NATS + go func(inst *vm.VM, attrs qmp.Attributes) { + stopErr := d.stopInstance(map[string]*vm.VM{inst.ID: inst}, isTerminate) + + if stopErr != nil { + slog.Error("Failed to "+strings.ToLower(action)+" instance", "err", stopErr, "id", inst.ID) + if err := d.TransitionState(inst, vm.StateError); err != nil { + slog.Error("Failed to transition to error state", "instanceId", inst.ID, "err", err) + } + } else { + d.Instances.Mu.Lock() + inst.Attributes = attrs + inst.LastNode = d.node + d.Instances.Mu.Unlock() + + if err := d.TransitionState(inst, finalState); err != nil { + slog.Error("Failed to transition to final state", "instanceId", inst.ID, "err", err) + } + slog.Info("Instance "+string(finalState), "id", inst.ID) + + // For stop (not terminate): release ownership to shared KV so any + // daemon can pick up the next StartInstance request. + if !isTerminate && d.jsManager != nil { + + // Write to shared KV first — if daemon crashes after this but + // before local cleanup, restoreInstances handles the overlap. + if err := d.jsManager.WriteStoppedInstance(inst.ID, inst); err != nil { + slog.Error("Failed to write stopped instance to shared KV, keeping local ownership", + "instanceId", inst.ID, "err", err) + } else { + // Unsubscribe from per-instance NATS topic + d.mu.Lock() + if sub, ok := d.natsSubscriptions[inst.ID]; ok { + if err := sub.Unsubscribe(); err != nil { + slog.Error("Failed to unsubscribe stopped instance", "instanceId", inst.ID, "err", err) + } + delete(d.natsSubscriptions, inst.ID) + } + d.mu.Unlock() + + // Remove from local instance map + d.Instances.Mu.Lock() + delete(d.Instances.VMS, inst.ID) + d.Instances.Mu.Unlock() + + // Persist local state without the stopped instance + if err := d.WriteState(); err != nil { + slog.Error("Failed to persist state after releasing stopped instance, re-adding to local map for consistency", + "instanceId", inst.ID, "err", err) + // Re-add to local map so in-memory state matches on-disk state. + // On next restart, restoreInstances will retry the migration. + d.Instances.Mu.Lock() + d.Instances.VMS[inst.ID] = inst + d.Instances.Mu.Unlock() + } else { + slog.Info("Released stopped instance ownership to shared KV", + "instanceId", inst.ID, "lastNode", d.node) + } + } + } + } + }(instance, command.Attributes) +} + +// handleEC2DescribeInstances processes incoming EC2 DescribeInstances requests +// This handler responds with all instances running on this node +func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) { + slog.Debug("Received message", "subject", msg.Subject, "data", string(msg.Data)) + + // Initialize describeInstancesInput before unmarshaling into it + describeInstancesInput := &ec2.DescribeInstancesInput{} + var errResp []byte + + errResp = utils.UnmarshalJsonPayload(describeInstancesInput, msg.Data) + + if errResp != nil { + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + slog.Error("Request does not match DescribeInstancesInput") + return + } + + slog.Info("Processing DescribeInstances request from this node") + + d.Instances.Mu.Lock() + defer d.Instances.Mu.Unlock() + + // Validate and filter instances if specific instance IDs were requested + instanceIDFilter := make(map[string]bool) + if len(describeInstancesInput.InstanceIds) > 0 { + for _, id := range describeInstancesInput.InstanceIds { + if id != nil && *id != "" { + if !strings.HasPrefix(*id, "i-") { + if err := msg.Respond(utils.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDMalformed)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + instanceIDFilter[*id] = true + } + } + } + + // Group instances by reservation ID (AWS returns instances grouped by reservation) + reservationMap := make(map[string]*ec2.Reservation) + + // Iterate through all instances on this node + for _, instance := range d.Instances.VMS { + // Skip if filtering by instance IDs and this instance is not in the filter + if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] { + continue + } + + // Use stored reservation metadata if available + if instance.Reservation != nil && instance.Instance != nil { + resID := "" + if instance.Reservation.ReservationId != nil { + resID = *instance.Reservation.ReservationId + } + + // Create reservation entry if it doesn't exist + if _, exists := reservationMap[resID]; !exists { + reservation := &ec2.Reservation{} + reservation.SetReservationId(resID) + if instance.Reservation.OwnerId != nil { + reservation.SetOwnerId(*instance.Reservation.OwnerId) + } + reservation.Instances = []*ec2.Instance{} + reservationMap[resID] = reservation + } + + // Update the instance state to current state + instanceCopy := *instance.Instance + instanceCopy.State = &ec2.InstanceState{} + + // Map internal status to EC2 state codes using the centralized mapping + if info, ok := vm.EC2StateCodes[instance.Status]; ok { + instanceCopy.State.SetCode(info.Code) + instanceCopy.State.SetName(info.Name) + } else { + slog.Warn("Instance has unmapped status, reporting as pending", + "instanceId", instance.ID, "status", string(instance.Status)) + instanceCopy.State.SetCode(0) + instanceCopy.State.SetName("pending") + } + + // Add instance to its reservation + reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy) + } + } + + // Convert map to slice + reservations := make([]*ec2.Reservation, 0, len(reservationMap)) + for _, reservation := range reservationMap { + reservations = append(reservations, reservation) + } + + // Create the response + output := &ec2.DescribeInstancesOutput{ + Reservations: reservations, + } + + // Respond to NATS with DescribeInstancesOutput + jsonResponse, err := json.Marshal(output) + if err != nil { + slog.Error("handleEC2DescribeInstances failed to marshal output", "err", err) + errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + if err := msg.Respond(jsonResponse); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + slog.Info("handleEC2DescribeInstances completed", "count", len(reservations)) +} + +// handleEC2DescribeInstanceTypes processes incoming EC2 DescribeInstanceTypes requests +// This handler responds with instance types that can currently be provisioned on this node +// based on available resources (CPU and memory not already allocated to running instances) +func (d *Daemon) handleEC2DescribeInstanceTypes(msg *nats.Msg) { + slog.Debug("Received message", "subject", msg.Subject) + + // Initialize input + describeInput := &ec2.DescribeInstanceTypesInput{} + var errResp []byte + + errResp = utils.UnmarshalJsonPayload(describeInput, msg.Data) + if errResp != nil { + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + slog.Error("Request does not match DescribeInstanceTypesInput") + return + } + + slog.Info("Processing DescribeInstanceTypes request from this node") + + // Check if "capacity" filter is set to "true" + showCapacity := false + for _, f := range describeInput.Filters { + if f.Name != nil && *f.Name == "capacity" { + for _, v := range f.Values { + if v != nil && *v == "true" { + showCapacity = true + break + } + } + } + } + + // Get instance types based on capacity and the showCapacity flag + filteredTypes := d.resourceMgr.GetAvailableInstanceTypeInfos(showCapacity) + + // Create the response + output := &ec2.DescribeInstanceTypesOutput{ + InstanceTypes: filteredTypes, + } + + // Respond to NATS + jsonResponse, err := json.Marshal(output) + if err != nil { + slog.Error("handleEC2DescribeInstanceTypes failed to marshal output", "err", err) + errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + if err := msg.Respond(jsonResponse); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + slog.Info("handleEC2DescribeInstanceTypes completed", "count", len(filteredTypes)) +} + +// startStoppedInstanceRequest is the payload for ec2.start topic +type startStoppedInstanceRequest struct { + InstanceID string `json:"instance_id"` +} + +// handleEC2StartStoppedInstance picks up a stopped instance from shared KV, +// re-launches it on this daemon node, and removes it from shared KV. +func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) { + respondWithError := func(errCode string) { + if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + } + + var req startStoppedInstanceRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("handleEC2StartStoppedInstance: failed to unmarshal request", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + if req.InstanceID == "" { + slog.Error("handleEC2StartStoppedInstance: missing instance_id") + respondWithError(awserrors.ErrorMissingParameter) + return + } + + if d.jsManager == nil { + slog.Error("handleEC2StartStoppedInstance: JetStream not available") + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Load instance from shared KV + instance, err := d.jsManager.LoadStoppedInstance(req.InstanceID) + if err != nil { + slog.Error("handleEC2StartStoppedInstance: failed to load stopped instance", "instanceId", req.InstanceID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + if instance == nil { + slog.Warn("handleEC2StartStoppedInstance: instance not found in shared KV", "instanceId", req.InstanceID) + respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) + return + } + + if instance.Status != vm.StateStopped { + slog.Error("handleEC2StartStoppedInstance: instance not in stopped state", "instanceId", req.InstanceID, "status", instance.Status) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + // Reset node-local fields that are stale after cross-node migration + instance.ResetNodeLocalState() + + // Allocate resources + instanceType, ok := d.resourceMgr.instanceTypes[instance.InstanceType] + if !ok { + slog.Error("handleEC2StartStoppedInstance: instance type not available on this node", + "instanceId", req.InstanceID, "instanceType", instance.InstanceType) + respondWithError(awserrors.ErrorInsufficientInstanceCapacity) + return + } + if err := d.resourceMgr.allocate(instanceType); err != nil { + slog.Error("handleEC2StartStoppedInstance: failed to allocate resources", "instanceId", req.InstanceID, "err", err) + respondWithError(awserrors.ErrorInsufficientInstanceCapacity) + return + } + + // Add instance to local map and clear stop attribute before launch + d.Instances.Mu.Lock() + d.Instances.VMS[instance.ID] = instance + instance.Attributes = qmp.Attributes{StartInstance: true} + d.Instances.Mu.Unlock() + + // Launch the instance infrastructure (QEMU, QMP, NATS subscriptions) + err = d.LaunchInstance(instance) + if err != nil { + slog.Error("handleEC2StartStoppedInstance: LaunchInstance failed", "instanceId", req.InstanceID, "err", err) + // Rollback: deallocate resources and remove from local map + if ok { + d.resourceMgr.deallocate(instanceType) + } + d.Instances.Mu.Lock() + delete(d.Instances.VMS, instance.ID) + d.Instances.Mu.Unlock() + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Remove from shared KV now that it's running locally. + // Retry once on failure — a stale KV entry risks duplicate starts. + if err := d.jsManager.DeleteStoppedInstance(req.InstanceID); err != nil { + slog.Warn("handleEC2StartStoppedInstance: first KV delete failed, retrying", + "instanceId", req.InstanceID, "err", err) + if retryErr := d.jsManager.DeleteStoppedInstance(req.InstanceID); retryErr != nil { + slog.Error("handleEC2StartStoppedInstance: KV delete failed after retry, instance is running locally but stale entry remains in shared KV", + "instanceId", req.InstanceID, "err", retryErr) + } + } + + slog.Info("Started stopped instance from shared KV", "instanceId", instance.ID, "node", d.node) + + if err := msg.Respond(fmt.Appendf(nil, `{"status":"running","instanceId":"%s"}`, instance.ID)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } +} + +// terminateStoppedInstanceRequest is the payload for ec2.terminate topic +type terminateStoppedInstanceRequest struct { + InstanceID string `json:"instance_id"` +} + +// handleEC2TerminateStoppedInstance picks up a stopped instance from shared KV, +// deletes its volumes, and removes it from shared KV. +func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) { + respondWithError := func(errCode string) { + if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + } + + var req terminateStoppedInstanceRequest + if err := json.Unmarshal(msg.Data, &req); err != nil { + slog.Error("handleEC2TerminateStoppedInstance: failed to unmarshal request", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + if req.InstanceID == "" { + slog.Error("handleEC2TerminateStoppedInstance: missing instance_id") + respondWithError(awserrors.ErrorMissingParameter) + return + } + + if d.jsManager == nil { + slog.Error("handleEC2TerminateStoppedInstance: JetStream not available") + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Load instance from shared KV + instance, err := d.jsManager.LoadStoppedInstance(req.InstanceID) + if err != nil { + slog.Error("handleEC2TerminateStoppedInstance: failed to load stopped instance", "instanceId", req.InstanceID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + if instance == nil { + slog.Warn("handleEC2TerminateStoppedInstance: instance not found in shared KV", "instanceId", req.InstanceID) + respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) + return + } + + if instance.Status != vm.StateStopped { + slog.Error("handleEC2TerminateStoppedInstance: instance not in stopped state", "instanceId", req.InstanceID, "status", instance.Status) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + // Delete volumes — no QEMU shutdown or unmount needed (already done during stop) + instance.EBSRequests.Mu.Lock() + for _, ebsRequest := range instance.EBSRequests.Requests { + // Internal volumes (EFI, cloud-init) are always cleaned up via ebs.delete + if ebsRequest.EFI || ebsRequest.CloudInit { + ebsDeleteData, err := json.Marshal(config.EBSDeleteRequest{Volume: ebsRequest.Name}) + if err != nil { + slog.Error("handleEC2TerminateStoppedInstance: failed to marshal ebs.delete request", "name", ebsRequest.Name, "err", err) + continue + } + deleteMsg, err := d.natsConn.Request("ebs.delete", ebsDeleteData, 30*time.Second) + if err != nil { + slog.Warn("handleEC2TerminateStoppedInstance: ebs.delete failed for internal volume", "name", ebsRequest.Name, "err", err) + } else { + slog.Info("handleEC2TerminateStoppedInstance: ebs.delete sent for internal volume", "name", ebsRequest.Name, "data", string(deleteMsg.Data)) + } + continue + } + + // User-visible volumes: respect DeleteOnTermination flag + if !ebsRequest.DeleteOnTermination { + slog.Info("handleEC2TerminateStoppedInstance: volume has DeleteOnTermination=false, skipping", "name", ebsRequest.Name) + continue + } + + slog.Info("handleEC2TerminateStoppedInstance: deleting volume with DeleteOnTermination=true", "name", ebsRequest.Name) + _, err := d.volumeService.DeleteVolume(&ec2.DeleteVolumeInput{ + VolumeId: &ebsRequest.Name, + }) + if err != nil { + slog.Error("handleEC2TerminateStoppedInstance: failed to delete volume", "name", ebsRequest.Name, "err", err) + } + } + instance.EBSRequests.Mu.Unlock() + + // Remove from shared KV + if err := d.jsManager.DeleteStoppedInstance(req.InstanceID); err != nil { + slog.Error("handleEC2TerminateStoppedInstance: failed to delete from shared KV", "instanceId", req.InstanceID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + slog.Info("Terminated stopped instance from shared KV", "instanceId", req.InstanceID) + + if err := msg.Respond(fmt.Appendf(nil, `{"status":"terminated","instanceId":"%s"}`, req.InstanceID)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } +} + +// handleEC2DescribeStoppedInstances returns stopped instances from shared KV. +func (d *Daemon) handleEC2DescribeStoppedInstances(msg *nats.Msg) { + respondWithError := func(errCode string) { + if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + } + + if d.jsManager == nil { + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Parse optional filters from the request + describeInput := &ec2.DescribeInstancesInput{} + if len(msg.Data) > 0 { + if errResp := utils.UnmarshalJsonPayload(describeInput, msg.Data); errResp != nil { + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + } + + // Build instance ID filter + instanceIDFilter := make(map[string]bool) + if len(describeInput.InstanceIds) > 0 { + for _, id := range describeInput.InstanceIds { + if id != nil { + instanceIDFilter[*id] = true + } + } + } + + instances, err := d.jsManager.ListStoppedInstances() + if err != nil { + slog.Error("handleEC2DescribeStoppedInstances: failed to list stopped instances", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Group instances by reservation ID + reservationMap := make(map[string]*ec2.Reservation) + + for _, instance := range instances { + // Apply instance ID filter + if len(instanceIDFilter) > 0 && !instanceIDFilter[instance.ID] { + continue + } + + if instance.Reservation != nil && instance.Instance != nil { + resID := "" + if instance.Reservation.ReservationId != nil { + resID = *instance.Reservation.ReservationId + } + + if _, exists := reservationMap[resID]; !exists { + reservation := &ec2.Reservation{} + reservation.SetReservationId(resID) + if instance.Reservation.OwnerId != nil { + reservation.SetOwnerId(*instance.Reservation.OwnerId) + } + reservation.Instances = []*ec2.Instance{} + reservationMap[resID] = reservation + } + + // Update the instance state + instanceCopy := *instance.Instance + instanceCopy.State = &ec2.InstanceState{} + if info, ok := vm.EC2StateCodes[instance.Status]; ok { + instanceCopy.State.SetCode(info.Code) + instanceCopy.State.SetName(info.Name) + } else { + instanceCopy.State.SetCode(80) + instanceCopy.State.SetName("stopped") + } + + reservationMap[resID].Instances = append(reservationMap[resID].Instances, &instanceCopy) + } + } + + reservations := make([]*ec2.Reservation, 0, len(reservationMap)) + for _, reservation := range reservationMap { + reservations = append(reservations, reservation) + } + + output := &ec2.DescribeInstancesOutput{ + Reservations: reservations, + } + + jsonResponse, err := json.Marshal(output) + if err != nil { + slog.Error("handleEC2DescribeStoppedInstances: failed to marshal output", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + if err := msg.Respond(jsonResponse); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + slog.Info("handleEC2DescribeStoppedInstances completed", "count", len(reservations)) +} + +// handleEC2ModifyInstanceAttribute modifies attributes of a stopped instance in shared KV. +// All supported attributes (InstanceType, UserData) require the instance to be stopped. +func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) { + respondWithError := func(errCode string) { + if err := msg.Respond(utils.GenerateErrorPayload(errCode)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + } + + var input ec2.ModifyInstanceAttributeInput + if err := json.Unmarshal(msg.Data, &input); err != nil { + slog.Error("handleEC2ModifyInstanceAttribute: failed to unmarshal request", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + if input.InstanceId == nil || *input.InstanceId == "" { + slog.Error("handleEC2ModifyInstanceAttribute: missing instance_id") + respondWithError(awserrors.ErrorMissingParameter) + return + } + + instanceID := *input.InstanceId + + if d.jsManager == nil { + slog.Error("handleEC2ModifyInstanceAttribute: JetStream not available") + respondWithError(awserrors.ErrorServerInternal) + return + } + + instance, err := d.jsManager.LoadStoppedInstance(instanceID) + if err != nil { + slog.Error("handleEC2ModifyInstanceAttribute: failed to load stopped instance", "instanceId", instanceID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + if instance == nil { + slog.Warn("handleEC2ModifyInstanceAttribute: instance not found in shared KV", "instanceId", instanceID) + respondWithError(awserrors.ErrorInvalidInstanceIDNotFound) + return + } + + if instance.Status != vm.StateStopped { + slog.Error("handleEC2ModifyInstanceAttribute: instance not in stopped state", "instanceId", instanceID, "status", instance.Status) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + // Apply the requested attribute change + if input.InstanceType != nil && input.InstanceType.Value != nil { + newType := *input.InstanceType.Value + if newType == "" { + slog.Error("handleEC2ModifyInstanceAttribute: empty instance type value", "instanceId", instanceID) + respondWithError(awserrors.ErrorInvalidInstanceAttributeValue) + return + } + if instance.Instance == nil { + slog.Error("handleEC2ModifyInstanceAttribute: instance.Instance is nil, data integrity issue", "instanceId", instanceID) + respondWithError(awserrors.ErrorServerInternal) + return + } + slog.Info("handleEC2ModifyInstanceAttribute: changing instance type", + "instanceId", instanceID, "oldType", instance.InstanceType, "newType", newType) + + instance.InstanceType = newType + instance.Config.InstanceType = newType + instance.Instance.InstanceType = aws.String(newType) + // Clear StateReason — resolves capacity-unavailable state from instance-type-missing bug + instance.Instance.StateReason = nil + } + + if input.UserData != nil && input.UserData.Value != nil { + slog.Info("handleEC2ModifyInstanceAttribute: changing user data", "instanceId", instanceID) + + // Value arrives as decoded bytes (JSON unmarshal handles base64 → []byte automatically) + instance.UserData = string(input.UserData.Value) + if instance.RunInstancesInput != nil { + instance.RunInstancesInput.UserData = aws.String(base64.StdEncoding.EncodeToString(input.UserData.Value)) + } + } + + if err := d.jsManager.WriteStoppedInstance(instanceID, instance); err != nil { + slog.Error("handleEC2ModifyInstanceAttribute: failed to write modified instance to KV", + "instanceId", instanceID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + slog.Info("handleEC2ModifyInstanceAttribute: completed successfully", "instanceId", instanceID) + + if err := msg.Respond([]byte(`{}`)); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } +} diff --git a/hive/daemon/daemon_handlers_key.go b/hive/daemon/daemon_handlers_key.go new file mode 100644 index 000000000..9b288a2cb --- /dev/null +++ b/hive/daemon/daemon_handlers_key.go @@ -0,0 +1,21 @@ +package daemon + +import ( + "github.com/nats-io/nats.go" +) + +func (d *Daemon) handleEC2CreateKeyPair(msg *nats.Msg) { + handleNATSRequest(msg, d.keyService.CreateKeyPair) +} + +func (d *Daemon) handleEC2DeleteKeyPair(msg *nats.Msg) { + handleNATSRequest(msg, d.keyService.DeleteKeyPair) +} + +func (d *Daemon) handleEC2DescribeKeyPairs(msg *nats.Msg) { + handleNATSRequest(msg, d.keyService.DescribeKeyPairs) +} + +func (d *Daemon) handleEC2ImportKeyPair(msg *nats.Msg) { + handleNATSRequest(msg, d.keyService.ImportKeyPair) +} diff --git a/hive/daemon/daemon_handlers_snapshot.go b/hive/daemon/daemon_handlers_snapshot.go new file mode 100644 index 000000000..a6fcf2819 --- /dev/null +++ b/hive/daemon/daemon_handlers_snapshot.go @@ -0,0 +1,21 @@ +package daemon + +import ( + "github.com/nats-io/nats.go" +) + +func (d *Daemon) handleEC2CreateSnapshot(msg *nats.Msg) { + handleNATSRequest(msg, d.snapshotService.CreateSnapshot) +} + +func (d *Daemon) handleEC2DescribeSnapshots(msg *nats.Msg) { + handleNATSRequest(msg, d.snapshotService.DescribeSnapshots) +} + +func (d *Daemon) handleEC2DeleteSnapshot(msg *nats.Msg) { + handleNATSRequest(msg, d.snapshotService.DeleteSnapshot) +} + +func (d *Daemon) handleEC2CopySnapshot(msg *nats.Msg) { + handleNATSRequest(msg, d.snapshotService.CopySnapshot) +} diff --git a/hive/daemon/daemon_handlers_tag.go b/hive/daemon/daemon_handlers_tag.go new file mode 100644 index 000000000..743db9026 --- /dev/null +++ b/hive/daemon/daemon_handlers_tag.go @@ -0,0 +1,17 @@ +package daemon + +import ( + "github.com/nats-io/nats.go" +) + +func (d *Daemon) handleEC2CreateTags(msg *nats.Msg) { + handleNATSRequest(msg, d.tagsService.CreateTags) +} + +func (d *Daemon) handleEC2DeleteTags(msg *nats.Msg) { + handleNATSRequest(msg, d.tagsService.DeleteTags) +} + +func (d *Daemon) handleEC2DescribeTags(msg *nats.Msg) { + handleNATSRequest(msg, d.tagsService.DescribeTags) +} diff --git a/hive/daemon/daemon_handlers_test.go b/hive/daemon/daemon_handlers_test.go index 562387af0..28f5a5456 100644 --- a/hive/daemon/daemon_handlers_test.go +++ b/hive/daemon/daemon_handlers_test.go @@ -2519,3 +2519,739 @@ func TestDelegateHandlers_EIGW(t *testing.T) { }) } } + +// --- handleEC2ModifyVolume success path --- + +func TestHandleEC2ModifyVolume_Success(t *testing.T) { + daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL) + + // Seed a volume in the store + volumeID := "vol-modify-success" + wrapper := struct { + VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` + }{ + VolumeConfig: viperblock.VolumeConfig{ + VolumeMetadata: viperblock.VolumeMetadata{ + VolumeID: volumeID, + SizeGiB: 10, + State: "available", + VolumeType: "gp3", + }, + }, + } + data, _ := json.Marshal(wrapper) + store.PutObject(&awss3.PutObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String(volumeID + "/config.json"), + Body: strings.NewReader(string(data)), + }) + + // Subscribe a dummy ebs.sync handler so the NATS Request doesn't time out + syncSub, err := daemon.natsConn.Subscribe("ebs.sync", func(msg *nats.Msg) { + _ = msg.Respond([]byte(`{}`)) + }) + require.NoError(t, err) + defer syncSub.Unsubscribe() + + sub, err := daemon.natsConn.QueueSubscribe("ec2.ModifyVolume", "hive-workers", daemon.handleEC2ModifyVolume) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.ModifyVolumeInput{ + VolumeId: aws.String(volumeID), + Size: aws.Int64(20), + } + reqData, _ := json.Marshal(input) + reply, err := daemon.natsConn.Request("ec2.ModifyVolume", reqData, 5*time.Second) + require.NoError(t, err) + + var output ec2.ModifyVolumeOutput + err = json.Unmarshal(reply.Data, &output) + require.NoError(t, err) + require.NotNil(t, output.VolumeModification) + assert.Equal(t, volumeID, *output.VolumeModification.VolumeId) + assert.Equal(t, int64(10), *output.VolumeModification.OriginalSize) + assert.Equal(t, int64(20), *output.VolumeModification.TargetSize) + assert.Equal(t, "completed", *output.VolumeModification.ModificationState) +} + +// --- handleEC2TerminateStoppedInstance with volumes --- + +func TestHandleEC2TerminateStoppedInstance_WithVolumes(t *testing.T) { + daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL) + + // Subscribe a dummy ebs.delete handler + ebsDeleteSub, err := daemon.natsConn.Subscribe("ebs.delete", func(msg *nats.Msg) { + _ = msg.Respond([]byte(`{"status":"deleted"}`)) + }) + require.NoError(t, err) + defer ebsDeleteSub.Unsubscribe() + + stoppedVM := &vm.VM{ + ID: "i-term-vol-001", + Status: vm.StateStopped, + InstanceType: getTestInstanceType(), + LastNode: "node-1", + Reservation: &ec2.Reservation{ + ReservationId: aws.String("r-term-vol-001"), + OwnerId: aws.String("123456789012"), + }, + Instance: &ec2.Instance{ + InstanceId: aws.String("i-term-vol-001"), + InstanceType: aws.String(getTestInstanceType()), + }, + } + // Add EFI, CloudInit, and a user volume with DeleteOnTermination + stoppedVM.EBSRequests.Requests = []config.EBSRequest{ + {Name: "vol-efi-001", EFI: true}, + {Name: "vol-ci-001", CloudInit: true}, + {Name: "vol-user-001", DeleteOnTermination: true}, + {Name: "vol-keep-001", DeleteOnTermination: false}, + } + + err = daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM) + require.NoError(t, err) + + sub, err := daemon.natsConn.QueueSubscribe("ec2.terminate", "hive-workers", daemon.handleEC2TerminateStoppedInstance) + require.NoError(t, err) + defer sub.Unsubscribe() + + reqData, _ := json.Marshal(map[string]string{"instance_id": "i-term-vol-001"}) + reply, err := daemon.natsConn.Request("ec2.terminate", reqData, 10*time.Second) + require.NoError(t, err) + + var resp map[string]string + err = json.Unmarshal(reply.Data, &resp) + require.NoError(t, err) + assert.Equal(t, "terminated", resp["status"]) + assert.Equal(t, "i-term-vol-001", resp["instanceId"]) + + loaded, err := daemon.jsManager.LoadStoppedInstance("i-term-vol-001") + require.NoError(t, err) + assert.Nil(t, loaded) +} + +// --- handleEC2DescribeInstanceTypes with capacity filter --- + +func TestHandleEC2DescribeInstanceTypes_CapacityFilter(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceTypes", "hive-workers", daemon.handleEC2DescribeInstanceTypes) + require.NoError(t, err) + defer sub.Unsubscribe() + + // Request with capacity=true filter + input := &ec2.DescribeInstanceTypesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("capacity"), + Values: []*string{aws.String("true")}, + }, + }, + } + reqData, _ := json.Marshal(input) + reply, err := daemon.natsConn.Request("ec2.DescribeInstanceTypes", reqData, 5*time.Second) + require.NoError(t, err) + + var output ec2.DescribeInstanceTypesOutput + err = json.Unmarshal(reply.Data, &output) + require.NoError(t, err) + // Should return instance types that fit the node's available capacity + assert.NotNil(t, output.InstanceTypes) +} + +func TestHandleEC2DescribeInstanceTypes_NoFilter(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + sub, err := daemon.natsConn.QueueSubscribe("ec2.DescribeInstanceTypes.nofilter", "hive-workers", daemon.handleEC2DescribeInstanceTypes) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.DescribeInstanceTypesInput{} + reqData, _ := json.Marshal(input) + reply, err := daemon.natsConn.Request("ec2.DescribeInstanceTypes.nofilter", reqData, 5*time.Second) + require.NoError(t, err) + + var output ec2.DescribeInstanceTypesOutput + err = json.Unmarshal(reply.Data, &output) + require.NoError(t, err) + assert.NotNil(t, output.InstanceTypes) + assert.Greater(t, len(output.InstanceTypes), 0) +} + +// --- handleEC2StartStoppedInstance: instance type not available --- + +func TestHandleEC2StartStoppedInstance_InstanceTypeNotAvailable(t *testing.T) { + daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL) + + stoppedVM := &vm.VM{ + ID: "i-start-badtype-001", + Status: vm.StateStopped, + InstanceType: "z99.nonexistent", + Instance: &ec2.Instance{ + InstanceId: aws.String("i-start-badtype-001"), + InstanceType: aws.String("z99.nonexistent"), + }, + } + err := daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM) + require.NoError(t, err) + t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(stoppedVM.ID) }) + + sub, err := daemon.natsConn.QueueSubscribe("ec2.start", "hive-workers", daemon.handleEC2StartStoppedInstance) + require.NoError(t, err) + defer sub.Unsubscribe() + + reqData, _ := json.Marshal(map[string]string{"instance_id": "i-start-badtype-001"}) + reply, err := daemon.natsConn.Request("ec2.start", reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + err = json.Unmarshal(reply.Data, &errResp) + require.NoError(t, err) + assert.Contains(t, errResp, "Code") +} + +// --- handleEC2CreateImage: running instance with valid root volume --- + +func TestHandleEC2CreateImage_RunningInstanceReachesService(t *testing.T) { + daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL) + + instanceID := "i-create-image-running" + rootVolumeID := "vol-root-img-001" + sourceImageID := "ami-source-001" + + // Seed a root volume config + wrapper := struct { + VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` + }{ + VolumeConfig: viperblock.VolumeConfig{ + VolumeMetadata: viperblock.VolumeMetadata{ + VolumeID: rootVolumeID, + SizeGiB: 8, + State: "in-use", + }, + }, + } + volData, _ := json.Marshal(wrapper) + store.PutObject(&awss3.PutObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String(rootVolumeID + "/config.json"), + Body: strings.NewReader(string(volData)), + }) + + daemon.Instances.Mu.Lock() + daemon.Instances.VMS[instanceID] = &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{ + InstanceId: aws.String(instanceID), + ImageId: aws.String(sourceImageID), + BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{ + { + DeviceName: aws.String("/dev/sda1"), + Ebs: &ec2.EbsInstanceBlockDevice{ + VolumeId: aws.String(rootVolumeID), + }, + }, + }, + }, + } + daemon.Instances.Mu.Unlock() + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.%s.CreateImage", instanceID), + daemon.handleEC2CreateImage, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.CreateImageInput{ + InstanceId: aws.String(instanceID), + Name: aws.String("test-image-snapshot"), + } + reqData, _ := json.Marshal(input) + reply, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.%s.CreateImage", instanceID), + reqData, + 5*time.Second, + ) + require.NoError(t, err) + // The service call may fail (no real S3 backend), but we've exercised + // the handler path up through service invocation: instance lookup, + // state validation, root volume extraction, params construction. + assert.NotEmpty(t, reply.Data) +} + +// --- handleAttachVolume: missing volume data --- + +func TestAttachVolume_MissingVolumeData(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + instanceID := "i-attach-missing-vol" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + // AttachVolume with nil AttachVolumeData + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + AttachVolume: true, + }, + AttachVolumeData: nil, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "InvalidParameterValue") +} + +func TestAttachVolume_InstanceNotRunning(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + instanceID := "i-attach-stopped" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateStopped, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + AttachVolume: true, + }, + AttachVolumeData: &qmp.AttachVolumeData{ + VolumeID: "vol-test-123", + }, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "IncorrectInstanceState") +} + +func TestAttachVolume_VolumeNotFound(t *testing.T) { + daemon, _ := createFullTestDaemonWithStore(t, sharedNATSURL) + + instanceID := "i-attach-vol-notfound" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + AttachVolume: true, + }, + AttachVolumeData: &qmp.AttachVolumeData{ + VolumeID: "vol-nonexistent-999", + }, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "InvalidVolume.NotFound") +} + +func TestAttachVolume_VolumeInUse(t *testing.T) { + daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL) + + instanceID := "i-attach-vol-inuse" + volumeID := "vol-in-use-001" + + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + daemon.Instances.VMS[instanceID] = instance + + // Seed a volume that is already in-use + wrapper := struct { + VolumeConfig viperblock.VolumeConfig `json:"VolumeConfig"` + }{ + VolumeConfig: viperblock.VolumeConfig{ + VolumeMetadata: viperblock.VolumeMetadata{ + VolumeID: volumeID, + SizeGiB: 10, + State: "in-use", + }, + }, + } + data, _ := json.Marshal(wrapper) + store.PutObject(&awss3.PutObjectInput{ + Bucket: aws.String("test-bucket"), + Key: aws.String(volumeID + "/config.json"), + Body: strings.NewReader(string(data)), + }) + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + AttachVolume: true, + }, + AttachVolumeData: &qmp.AttachVolumeData{ + VolumeID: volumeID, + }, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "VolumeInUse") +} + +// --- handleEC2Events: detach volume validation --- + +func TestDetachVolume_MissingVolumeData(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + instanceID := "i-detach-missing" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + DetachVolume: true, + }, + DetachVolumeData: nil, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "InvalidParameterValue") +} + +func TestDetachVolume_InstanceNotRunning(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + instanceID := "i-detach-stopped" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateStopped, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + DetachVolume: true, + }, + DetachVolumeData: &qmp.DetachVolumeData{ + VolumeID: "vol-test-123", + }, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "IncorrectInstanceState") +} + +func TestDetachVolume_VolumeNotAttached(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + instanceID := "i-detach-notattached" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + DetachVolume: true, + }, + DetachVolumeData: &qmp.DetachVolumeData{ + VolumeID: "vol-not-attached-999", + }, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "IncorrectState") +} + +func TestDetachVolume_BootVolumeRejected(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + instanceID := "i-detach-boot" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + instance.EBSRequests.Requests = []config.EBSRequest{ + {Name: "vol-boot-001", Boot: true, DeviceName: "/dev/sda1"}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + DetachVolume: true, + }, + DetachVolumeData: &qmp.DetachVolumeData{ + VolumeID: "vol-boot-001", + }, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "OperationNotPermitted") +} + +func TestDetachVolume_DeviceMismatch(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + instanceID := "i-detach-devmismatch" + instance := &vm.VM{ + ID: instanceID, + Status: vm.StateRunning, + InstanceType: getTestInstanceType(), + Instance: &ec2.Instance{}, + QMPClient: &qmp.QMPClient{}, + } + instance.EBSRequests.Requests = []config.EBSRequest{ + {Name: "vol-mismatch-001", DeviceName: "/dev/sdf"}, + } + daemon.Instances.VMS[instanceID] = instance + + sub, err := daemon.natsConn.Subscribe( + fmt.Sprintf("ec2.cmd.%s", instanceID), + daemon.handleEC2Events, + ) + require.NoError(t, err) + defer sub.Unsubscribe() + + command := qmp.Command{ + ID: instanceID, + Attributes: qmp.Attributes{ + DetachVolume: true, + }, + DetachVolumeData: &qmp.DetachVolumeData{ + VolumeID: "vol-mismatch-001", + Device: "/dev/sdg", + }, + } + cmdData, _ := json.Marshal(command) + + resp, err := daemon.natsConn.Request( + fmt.Sprintf("ec2.cmd.%s", instanceID), + cmdData, + 5*time.Second, + ) + require.NoError(t, err) + assert.Contains(t, string(resp.Data), "InvalidParameterValue") +} + +// --- handleEC2RunInstances: insufficient capacity --- + +func TestHandleEC2RunInstances_InsufficientCapacity(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + sub, err := daemon.natsConn.QueueSubscribe("ec2.RunInstances", "hive-workers", daemon.handleEC2RunInstances) + require.NoError(t, err) + defer sub.Unsubscribe() + + // Request a very large instance count that can't be satisfied + input := &ec2.RunInstancesInput{ + ImageId: aws.String("ami-test"), + InstanceType: aws.String(getTestInstanceType()), + MinCount: aws.Int64(9999), + MaxCount: aws.Int64(9999), + } + reqData, _ := json.Marshal(input) + reply, err := daemon.natsConn.Request("ec2.RunInstances", reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + err = json.Unmarshal(reply.Data, &errResp) + require.NoError(t, err) + assert.Contains(t, errResp, "Code") +} + +func TestHandleEC2RunInstances_UnsupportedInstanceType(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + sub, err := daemon.natsConn.QueueSubscribe("ec2.RunInstances.badtype", "hive-workers", daemon.handleEC2RunInstances) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.RunInstancesInput{ + ImageId: aws.String("ami-test"), + InstanceType: aws.String("z99.nonexistent"), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + } + reqData, _ := json.Marshal(input) + reply, err := daemon.natsConn.Request("ec2.RunInstances.badtype", reqData, 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + err = json.Unmarshal(reply.Data, &errResp) + require.NoError(t, err) + assert.Contains(t, errResp, "Code") +} + +func TestHandleEC2RunInstances_MalformedInput(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + sub, err := daemon.natsConn.QueueSubscribe("ec2.RunInstances.bad", "hive-workers", daemon.handleEC2RunInstances) + require.NoError(t, err) + defer sub.Unsubscribe() + + reply, err := daemon.natsConn.Request("ec2.RunInstances.bad", []byte(`{not valid}`), 5*time.Second) + require.NoError(t, err) + + var errResp map[string]any + err = json.Unmarshal(reply.Data, &errResp) + require.NoError(t, err) + assert.Contains(t, errResp, "Code") +} + +// --- handleEC2DescribeInstances: malformed instance ID --- + +func TestHandleEC2DescribeInstances_MalformedInstanceID(t *testing.T) { + daemon := createFullTestDaemon(t, sharedNATSURL) + + sub, err := daemon.natsConn.Subscribe("ec2.DescribeInstances.malformed", daemon.handleEC2DescribeInstances) + require.NoError(t, err) + defer sub.Unsubscribe() + + input := &ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String("not-an-instance-id")}, + } + reqData, _ := json.Marshal(input) + reply, err := daemon.natsConn.Request("ec2.DescribeInstances.malformed", reqData, 5*time.Second) + require.NoError(t, err) + + assert.Contains(t, string(reply.Data), "InvalidInstanceID.Malformed") +} diff --git a/hive/daemon/daemon_handlers_volume.go b/hive/daemon/daemon_handlers_volume.go new file mode 100644 index 000000000..95992b36b --- /dev/null +++ b/hive/daemon/daemon_handlers_volume.go @@ -0,0 +1,492 @@ +package daemon + +import ( + "encoding/json" + "fmt" + "log/slog" + "time" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/hive/hive/awserrors" + "github.com/mulgadc/hive/hive/config" + "github.com/mulgadc/hive/hive/qmp" + "github.com/mulgadc/hive/hive/utils" + "github.com/mulgadc/hive/hive/vm" + "github.com/nats-io/nats.go" +) + +// handleAttachVolume performs a three-phase hot-plug: +// +// Phase 1: ebs.mount via NATS (rolls back with ebs.unmount) +// Phase 2: QMP blockdev-add (rolls back Phase 1) +// Phase 3: QMP device_add (rolls back Phase 2 + Phase 1) +func (d *Daemon) handleAttachVolume(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { + slog.Info("Attaching volume to instance", "instanceId", command.ID) + + // Validate AttachVolumeData + if command.AttachVolumeData == nil || command.AttachVolumeData.VolumeID == "" { + slog.Error("AttachVolume: missing attach volume data") + respondWithError(awserrors.ErrorInvalidParameterValue) + return + } + + volumeID := command.AttachVolumeData.VolumeID + device := command.AttachVolumeData.Device + + // Validate instance is running + d.Instances.Mu.Lock() + status := instance.Status + d.Instances.Mu.Unlock() + + if status != vm.StateRunning { + slog.Error("AttachVolume: instance not running", "instanceId", command.ID, "status", status) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + // Validate volume exists and is available + volCfg, err := d.volumeService.GetVolumeConfig(volumeID) + if err != nil { + slog.Error("AttachVolume: failed to get volume config", "volumeId", volumeID, "err", err) + respondWithError(awserrors.ErrorInvalidVolumeNotFound) + return + } + + if volCfg.VolumeMetadata.State != "available" { + slog.Error("AttachVolume: volume not available", "volumeId", volumeID, "state", volCfg.VolumeMetadata.State) + respondWithError(awserrors.ErrorVolumeInUse) + return + } + + // Check AZ compatibility — volume and instance must be in the same AZ + if volCfg.VolumeMetadata.AvailabilityZone != "" && d.config.AZ != "" && + volCfg.VolumeMetadata.AvailabilityZone != d.config.AZ { + slog.Error("AttachVolume: volume and instance are in different AZs", + "volumeId", volumeID, "volumeAZ", volCfg.VolumeMetadata.AvailabilityZone, + "instanceAZ", d.config.AZ) + respondWithError(awserrors.ErrorInvalidVolumeZoneMismatch) + return + } + + // Determine device name + if device == "" { + d.Instances.Mu.Lock() + device = nextAvailableDevice(instance) + d.Instances.Mu.Unlock() + if device == "" { + slog.Error("AttachVolume: no available device names") + respondWithError(awserrors.ErrorAttachmentLimitExceeded) + return + } + } + + // Create EBS mount request + ebsRequest := config.EBSRequest{ + Name: volumeID, + DeviceName: device, + } + + ebsMountData, err := json.Marshal(ebsRequest) + if err != nil { + slog.Error("AttachVolume: failed to marshal ebs.mount request", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + mountReply, err := d.natsConn.Request(d.ebsTopic("mount"), ebsMountData, 30*time.Second) + if err != nil { + slog.Error("AttachVolume: ebs.mount failed", "volumeId", volumeID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + var mountResp config.EBSMountResponse + if err := json.Unmarshal(mountReply.Data, &mountResp); err != nil { + slog.Error("AttachVolume: failed to unmarshal mount response", "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + + if mountResp.Error != "" { + slog.Error("AttachVolume: mount returned error", "volumeId", volumeID, "err", mountResp.Error) + respondWithError(awserrors.ErrorServerInternal) + return + } + + nbdURI := mountResp.URI + if nbdURI == "" { + slog.Error("AttachVolume: mount response has empty URI", "volumeId", volumeID) + d.rollbackEBSMount(ebsRequest) + respondWithError(awserrors.ErrorServerInternal) + return + } + ebsRequest.NBDURI = nbdURI + + // Parse NBDURI for QMP blockdev-add + serverType, socketPath, nbdHost, nbdPort, err := utils.ParseNBDURI(nbdURI) + if err != nil { + slog.Error("AttachVolume: failed to parse NBDURI", "uri", nbdURI, "err", err) + // Rollback: unmount + d.rollbackEBSMount(ebsRequest) + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Build QMP server argument + var serverArg map[string]any + if serverType == "unix" { + serverArg = map[string]any{"type": "unix", "path": socketPath} + } else { + serverArg = map[string]any{"type": "inet", "host": nbdHost, "port": fmt.Sprintf("%d", nbdPort)} + } + + nodeName := fmt.Sprintf("nbd-%s", volumeID) + deviceID := fmt.Sprintf("vdisk-%s", volumeID) + iothreadID := fmt.Sprintf("ioth-%s", volumeID) + + // QMP object-add: create iothread for this volume + iothreadCmd := qmp.QMPCommand{ + Execute: "object-add", + Arguments: map[string]any{ + "qom-type": "iothread", + "id": iothreadID, + }, + } + _, err = d.SendQMPCommand(instance.QMPClient, iothreadCmd, instance.ID) + if err != nil { + slog.Error("AttachVolume: QMP object-add iothread failed", "volumeId", volumeID, "err", err) + d.rollbackEBSMount(ebsRequest) + respondWithError(awserrors.ErrorServerInternal) + return + } + + // QMP blockdev-add + blockdevCmd := qmp.QMPCommand{ + Execute: "blockdev-add", + Arguments: map[string]any{ + "node-name": nodeName, + "driver": "nbd", + "server": serverArg, + "export": "", + "read-only": false, + }, + } + + _, err = d.SendQMPCommand(instance.QMPClient, blockdevCmd, instance.ID) + if err != nil { + slog.Error("AttachVolume: QMP blockdev-add failed", "volumeId", volumeID, "err", err) + d.rollbackEBSMount(ebsRequest) + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Determine which hotplug root port to use based on device letter. + // /dev/sdf -> hotplug1, /dev/sdg -> hotplug2, etc. + hotplugBus := "" + if len(device) > 0 { + letter := device[len(device)-1] + if letter >= 'f' && letter <= 'p' { + hotplugBus = fmt.Sprintf("hotplug%d", letter-'f'+1) + } + } + + // QMP device_add + deviceAddArgs := map[string]any{ + "driver": "virtio-blk-pci", + "id": deviceID, + "drive": nodeName, + "iothread": iothreadID, + } + if hotplugBus != "" { + deviceAddArgs["bus"] = hotplugBus + } + deviceAddCmd := qmp.QMPCommand{ + Execute: "device_add", + Arguments: deviceAddArgs, + } + + _, err = d.SendQMPCommand(instance.QMPClient, deviceAddCmd, instance.ID) + if err != nil { + slog.Error("AttachVolume: QMP device_add failed, rolling back blockdev", "volumeId", volumeID, "err", err) + // Rollback: blockdev-del then unmount + if _, delErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ + Execute: "blockdev-del", + Arguments: map[string]any{"node-name": nodeName}, + }, instance.ID); delErr != nil { + slog.Error("AttachVolume: rollback blockdev-del failed, skipping EBS unmount", "volumeId", volumeID, "err", delErr) + } else { + d.rollbackEBSMount(ebsRequest) + } + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Update instance state: replace existing entry for this volume (handles + // stop/start cycles that keep stale entries) or append a new one. + instance.EBSRequests.Mu.Lock() + replaced := false + for i, req := range instance.EBSRequests.Requests { + if req.Name == volumeID { + instance.EBSRequests.Requests[i] = ebsRequest + replaced = true + break + } + } + if !replaced { + instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, ebsRequest) + } + instance.EBSRequests.Mu.Unlock() + + // Update BlockDeviceMappings on the ec2.Instance + d.Instances.Mu.Lock() + if instance.Instance != nil { + now := time.Now() + mapping := &ec2.InstanceBlockDeviceMapping{} + mapping.SetDeviceName(device) + mapping.Ebs = &ec2.EbsInstanceBlockDevice{} + mapping.Ebs.SetVolumeId(volumeID) + mapping.Ebs.SetAttachTime(now) + mapping.Ebs.SetDeleteOnTermination(false) + mapping.Ebs.SetStatus("attached") + instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping) + } + d.Instances.Mu.Unlock() + + // Update volume metadata in S3 + if err := d.volumeService.UpdateVolumeState(volumeID, "in-use", command.ID, device); err != nil { + slog.Error("AttachVolume: failed to update volume metadata", "volumeId", volumeID, "err", err) + } + + // Persist state + if err := d.WriteState(); err != nil { + slog.Error("AttachVolume: failed to write state", "err", err) + } + + d.respondWithVolumeAttachment(msg, respondWithError, volumeID, command.ID, device, "attached") + slog.Info("Volume attached successfully", "volumeId", volumeID, "instanceId", command.ID, "device", device) +} + +// handleDetachVolume performs a three-phase hot-unplug (reverse of attach): +// +// Phase 1: QMP device_del (remove guest device) +// Phase 2: QMP blockdev-del (remove block node) +// Phase 3: ebs.unmount NATS (stop NBD server) +func (d *Daemon) handleDetachVolume(msg *nats.Msg, command qmp.Command, instance *vm.VM, respondWithError func(string)) { + slog.Info("Detaching volume from instance", "instanceId", command.ID) + + // Validate DetachVolumeData + if command.DetachVolumeData == nil || command.DetachVolumeData.VolumeID == "" { + slog.Error("DetachVolume: missing detach volume data") + respondWithError(awserrors.ErrorInvalidParameterValue) + return + } + + volumeID := command.DetachVolumeData.VolumeID + device := command.DetachVolumeData.Device + force := command.DetachVolumeData.Force + + // Validate instance is running + d.Instances.Mu.Lock() + status := instance.Status + d.Instances.Mu.Unlock() + + if status != vm.StateRunning { + slog.Error("DetachVolume: instance not running", "instanceId", command.ID, "status", status) + respondWithError(awserrors.ErrorIncorrectInstanceState) + return + } + + // Find the volume in EBSRequests + instance.EBSRequests.Mu.Lock() + var ebsReq config.EBSRequest + found := false + for _, req := range instance.EBSRequests.Requests { + if req.Name == volumeID { + ebsReq = req + found = true + break + } + } + instance.EBSRequests.Mu.Unlock() + + if !found { + slog.Error("DetachVolume: volume not attached to instance", "volumeId", volumeID, "instanceId", command.ID) + respondWithError(awserrors.ErrorIncorrectState) + return + } + + // Reject detaching boot/EFI/CloudInit volumes + if ebsReq.Boot || ebsReq.EFI || ebsReq.CloudInit { + slog.Error("DetachVolume: cannot detach boot/EFI/CloudInit volume", "volumeId", volumeID) + respondWithError(awserrors.ErrorOperationNotPermitted) + return + } + + // Optional device cross-check + if device != "" && ebsReq.DeviceName != "" && device != ebsReq.DeviceName { + slog.Error("DetachVolume: device mismatch", "requested", device, "actual", ebsReq.DeviceName) + respondWithError(awserrors.ErrorInvalidParameterValue) + return + } + + deviceID := fmt.Sprintf("vdisk-%s", volumeID) + nodeName := fmt.Sprintf("nbd-%s", volumeID) + iothreadID := fmt.Sprintf("ioth-%s", volumeID) + + // Phase 1: QMP device_del (remove guest device) + _, err := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ + Execute: "device_del", + Arguments: map[string]any{"id": deviceID}, + }, instance.ID) + if err != nil { + if !force { + slog.Error("DetachVolume: QMP device_del failed", "volumeId", volumeID, "err", err) + respondWithError(awserrors.ErrorServerInternal) + return + } + slog.Warn("DetachVolume: QMP device_del failed (force=true, continuing)", "volumeId", volumeID, "err", err) + } + + // Brief pause for guest to acknowledge PCI removal + time.Sleep(d.detachDelay) + + // Phase 2: QMP blockdev-del (remove block node) + _, blockdevErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ + Execute: "blockdev-del", + Arguments: map[string]any{"node-name": nodeName}, + }, instance.ID) + if blockdevErr != nil { + // Block node still referenced by QEMU; do not clean up state or unmount — + // tearing down the NBD server would crash the VM, and removing metadata + // would allow the volume to be double-attached. + slog.Error("DetachVolume: QMP blockdev-del failed, leaving volume state intact", "volumeId", volumeID, "err", blockdevErr) + respondWithError(awserrors.ErrorServerInternal) + return + } + + // Phase 2b: QMP object-del (remove iothread, best-effort) + _, iothreadErr := d.SendQMPCommand(instance.QMPClient, qmp.QMPCommand{ + Execute: "object-del", + Arguments: map[string]any{"id": iothreadID}, + }, instance.ID) + if iothreadErr != nil { + slog.Warn("DetachVolume: QMP object-del iothread failed (non-fatal)", "volumeId", volumeID, "err", iothreadErr) + } + + // Phase 3: ebs.unmount via NATS (best-effort) + d.rollbackEBSMount(ebsReq) + + // State cleanup: remove volume from EBSRequests (search by name to avoid stale index) + instance.EBSRequests.Mu.Lock() + for i, req := range instance.EBSRequests.Requests { + if req.Name == volumeID { + instance.EBSRequests.Requests = append(instance.EBSRequests.Requests[:i], instance.EBSRequests.Requests[i+1:]...) + break + } + } + instance.EBSRequests.Mu.Unlock() + + // Remove from BlockDeviceMappings + d.Instances.Mu.Lock() + if instance.Instance != nil { + filtered := make([]*ec2.InstanceBlockDeviceMapping, 0, len(instance.Instance.BlockDeviceMappings)) + for _, bdm := range instance.Instance.BlockDeviceMappings { + if bdm.Ebs != nil && bdm.Ebs.VolumeId != nil && *bdm.Ebs.VolumeId == volumeID { + continue + } + filtered = append(filtered, bdm) + } + instance.Instance.BlockDeviceMappings = filtered + } + d.Instances.Mu.Unlock() + + // Update volume metadata to "available" + if err := d.volumeService.UpdateVolumeState(volumeID, "available", "", ""); err != nil { + slog.Error("DetachVolume: failed to update volume metadata", "volumeId", volumeID, "err", err) + } + + // Persist state + if err := d.WriteState(); err != nil { + slog.Error("DetachVolume: failed to write state", "err", err) + } + + d.respondWithVolumeAttachment(msg, respondWithError, volumeID, command.ID, ebsReq.DeviceName, "detaching") + slog.Info("Volume detached successfully", "volumeId", volumeID, "instanceId", command.ID) +} + +func (d *Daemon) handleEC2CreateVolume(msg *nats.Msg) { + handleNATSRequest(msg, d.volumeService.CreateVolume) +} + +func (d *Daemon) handleEC2DescribeVolumes(msg *nats.Msg) { + handleNATSRequest(msg, d.volumeService.DescribeVolumes) +} + +func (d *Daemon) handleEC2DescribeVolumeStatus(msg *nats.Msg) { + handleNATSRequest(msg, d.volumeService.DescribeVolumeStatus) +} + +// handleEC2ModifyVolume processes incoming EC2 ModifyVolume requests +func (d *Daemon) handleEC2ModifyVolume(msg *nats.Msg) { + slog.Debug("Received message", "subject", msg.Subject) + slog.Debug("Message data", "data", string(msg.Data)) + + modifyVolumeInput := &ec2.ModifyVolumeInput{} + var errResp []byte + + errResp = utils.UnmarshalJsonPayload(modifyVolumeInput, msg.Data) + + if errResp != nil { + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + slog.Error("Request does not match ModifyVolumeInput") + return + } + + slog.Info("Processing ModifyVolume request", "volumeId", modifyVolumeInput.VolumeId) + + output, err := d.volumeService.ModifyVolume(modifyVolumeInput) + + if err != nil { + slog.Error("handleEC2ModifyVolume service.ModifyVolume failed", "err", err) + errResp = utils.GenerateErrorPayload(err.Error()) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + + jsonResponse, err := json.Marshal(output) + if err != nil { + slog.Error("handleEC2ModifyVolume failed to marshal output", "err", err) + errResp = utils.GenerateErrorPayload(awserrors.ErrorServerInternal) + if err := msg.Respond(errResp); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + return + } + if err := msg.Respond(jsonResponse); err != nil { + slog.Error("Failed to respond to NATS request", "err", err) + } + + // Notify viperblockd to reload state after volume modification (e.g. resize) + if modifyVolumeInput.VolumeId != nil { + syncData, err := json.Marshal(config.EBSSyncRequest{Volume: *modifyVolumeInput.VolumeId}) + if err != nil { + slog.Error("failed to marshal ebs.sync request", "volumeId", *modifyVolumeInput.VolumeId, "err", err) + } else { + _, syncErr := d.natsConn.Request("ebs.sync", syncData, 5*time.Second) + if syncErr != nil { + slog.Warn("ebs.sync notification failed (volume may not be mounted)", + "volumeId", *modifyVolumeInput.VolumeId, "err", syncErr) + } + } + } + + slog.Info("handleEC2ModifyVolume completed", "volumeId", modifyVolumeInput.VolumeId) +} + +func (d *Daemon) handleEC2DeleteVolume(msg *nats.Msg) { + handleNATSRequest(msg, d.volumeService.DeleteVolume) +} diff --git a/hive/daemon/daemon_handlers_vpc.go b/hive/daemon/daemon_handlers_vpc.go new file mode 100644 index 000000000..fe9647118 --- /dev/null +++ b/hive/daemon/daemon_handlers_vpc.go @@ -0,0 +1,41 @@ +package daemon + +import ( + "github.com/nats-io/nats.go" +) + +func (d *Daemon) handleEC2CreateVpc(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.CreateVpc) +} + +func (d *Daemon) handleEC2DeleteVpc(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.DeleteVpc) +} + +func (d *Daemon) handleEC2DescribeVpcs(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.DescribeVpcs) +} + +func (d *Daemon) handleEC2CreateSubnet(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.CreateSubnet) +} + +func (d *Daemon) handleEC2DeleteSubnet(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.DeleteSubnet) +} + +func (d *Daemon) handleEC2DescribeSubnets(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.DescribeSubnets) +} + +func (d *Daemon) handleEC2CreateNetworkInterface(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.CreateNetworkInterface) +} + +func (d *Daemon) handleEC2DeleteNetworkInterface(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.DeleteNetworkInterface) +} + +func (d *Daemon) handleEC2DescribeNetworkInterfaces(msg *nats.Msg) { + handleNATSRequest(msg, d.vpcService.DescribeNetworkInterfaces) +} diff --git a/hive/daemon/daemon_test.go b/hive/daemon/daemon_test.go index d22097e3b..619a3be82 100644 --- a/hive/daemon/daemon_test.go +++ b/hive/daemon/daemon_test.go @@ -1331,24 +1331,6 @@ func TestRunInstances_CountValidation(t *testing.T) { t.Logf("Error response: %v", errResp) }) - t.Run("MinCount_zero", func(t *testing.T) { - input := &ec2.RunInstancesInput{ - ImageId: aws.String("ami-test"), - InstanceType: aws.String(instanceType), - MinCount: aws.Int64(0), // Invalid - MaxCount: aws.Int64(1), - } - inputJSON, _ := json.Marshal(input) - - resp, err := daemon.natsConn.Request(topic, inputJSON, 5*time.Second) - require.NoError(t, err) - - var errResp map[string]any - err = json.Unmarshal(resp.Data, &errResp) - require.NoError(t, err) - assert.Contains(t, errResp, "Code") - }) - t.Run("MaxCount_zero", func(t *testing.T) { input := &ec2.RunInstancesInput{ ImageId: aws.String("ami-test"), diff --git a/hive/handlers/ec2/igw/service_nats_test.go b/hive/handlers/ec2/igw/service_nats_test.go index 25f8bff29..dbca46218 100644 --- a/hive/handlers/ec2/igw/service_nats_test.go +++ b/hive/handlers/ec2/igw/service_nats_test.go @@ -19,11 +19,11 @@ func setupNATSIGWServiceTest(t *testing.T) (IGWService, *IGWServiceImpl) { backend, nc := setupTestIGWService(t) topics := map[string]func(*nats.Msg){ - "ec2.CreateInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateInternetGateway) }, - "ec2.DeleteInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteInternetGateway) }, + "ec2.CreateInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateInternetGateway) }, + "ec2.DeleteInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteInternetGateway) }, "ec2.DescribeInternetGateways": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DescribeInternetGateways) }, - "ec2.AttachInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.AttachInternetGateway) }, - "ec2.DetachInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DetachInternetGateway) }, + "ec2.AttachInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.AttachInternetGateway) }, + "ec2.DetachInternetGateway": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DetachInternetGateway) }, } for topic, handler := range topics { diff --git a/hive/handlers/ec2/vpc/service_nats_test.go b/hive/handlers/ec2/vpc/service_nats_test.go index b027a7485..f66003c7c 100644 --- a/hive/handlers/ec2/vpc/service_nats_test.go +++ b/hive/handlers/ec2/vpc/service_nats_test.go @@ -22,14 +22,14 @@ func setupNATSVPCServiceTest(t *testing.T) (VPCService, *VPCServiceImpl, *nats.C // Subscribe the backend handlers on NATS topics topics := map[string]func(*nats.Msg){ - "ec2.CreateVpc": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateVpc) }, - "ec2.DeleteVpc": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteVpc) }, - "ec2.DescribeVpcs": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DescribeVpcs) }, - "ec2.CreateSubnet": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateSubnet) }, - "ec2.DeleteSubnet": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteSubnet) }, - "ec2.DescribeSubnets": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DescribeSubnets) }, - "ec2.CreateNetworkInterface": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateNetworkInterface) }, - "ec2.DeleteNetworkInterface": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteNetworkInterface) }, + "ec2.CreateVpc": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateVpc) }, + "ec2.DeleteVpc": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteVpc) }, + "ec2.DescribeVpcs": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DescribeVpcs) }, + "ec2.CreateSubnet": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateSubnet) }, + "ec2.DeleteSubnet": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteSubnet) }, + "ec2.DescribeSubnets": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DescribeSubnets) }, + "ec2.CreateNetworkInterface": func(msg *nats.Msg) { handleNATSMsg(msg, backend.CreateNetworkInterface) }, + "ec2.DeleteNetworkInterface": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DeleteNetworkInterface) }, "ec2.DescribeNetworkInterfaces": func(msg *nats.Msg) { handleNATSMsg(msg, backend.DescribeNetworkInterfaces) }, } diff --git a/scripts/diff-coverage.sh b/scripts/diff-coverage.sh index 701c69b13..b51645165 100755 --- a/scripts/diff-coverage.sh +++ b/scripts/diff-coverage.sh @@ -64,8 +64,10 @@ TMPDIR=$(mktemp -d) trap 'rm -rf "$TMPDIR"' EXIT # --- Step 1: Extract added line numbers from diff --- -# Only non-test .go files that were added or modified -git diff "${BASE_REF}" HEAD --unified=0 --diff-filter=AM -- '*.go' ':!*_test.go' | awk ' +# Only non-test .go files that were added or modified. +# -M enables rename detection so file splits/moves only count truly changed lines. +# -C enables copy detection for the same reason (file split = partial copy). +git diff "${BASE_REF}" HEAD --unified=0 -M -C --diff-filter=AM -- '*.go' ':!*_test.go' | awk ' /^\+\+\+ / { file = substr($2, 3) next