From 8a94886b13907f7106bb08e19f8bcab37a2d5e1d Mon Sep 17 00:00:00 2001 From: Julian Sommer <115516918+juliansommer@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:48:08 +1000 Subject: [PATCH] feat: cloud-init EC2 datasource (#411) --- .github/workflows/e2e-suites.yml | 22 +- .github/workflows/e2e.yml | 5 +- .golangci.yml | 1 - cmd/spinifex/cmd/admin.go | 65 ++ cmd/spinifex/cmd/admin_cabake_test.go | 67 ++ docs/COMMANDS.md | 2 +- docs/DESIGN.md | 3 +- docs/compute/launching-instances/README.md | 2 +- .../setting-up-your-cluster/README.md | 2 +- .../SMCI-MIG-orchestration/README.md | 2 +- .../bastion-private-subnet/README.md | 8 +- .../bastion-private-subnet/main.tf | 16 +- docs/terraform-workbooks/nginx-alb/README.md | 2 +- .../nginx-webserver/README.md | 4 +- .../nginx-webserver/main.tf | 4 +- docs/terraform-workbooks/s3-webapp/README.md | 6 +- docs/terraform-workbooks/s3-webapp/main.tf | 4 +- go.mod | 3 +- go.sum | 2 - scripts/demo/deploy.sh | 4 +- scripts/images/eks-node/manifest.conf | 14 +- scripts/images/eks-node/mulga-mgmt-net.initd | 19 + scripts/images/eks-node/mulga-mgmt-net.sh | 95 +++ scripts/images/eks-node/setup.sh | 1 + scripts/launch-gpu.sh | 4 +- scripts/run-bench.sh | 6 +- scripts/smoke-test.sh | 10 +- spinifex/daemon/daemon_system_instance.go | 17 +- spinifex/daemon/daemon_test.go | 46 +- spinifex/daemon/resource.go | 4 +- spinifex/daemon/vm_adapters.go | 15 +- .../handlers/ec2/instance/helpers_test.go | 92 --- .../handlers/ec2/instance/service_impl.go | 578 +----------------- .../ec2/instance/service_impl_test.go | 449 +------------- spinifex/handlers/ec2/volume/service_impl.go | 5 +- spinifex/handlers/imds/metadata.go | 55 +- spinifex/handlers/imds/metadata_test.go | 106 +++- spinifex/handlers/imds/tap_responder.go | 63 +- spinifex/handlers/imds/tap_responder_test.go | 42 ++ spinifex/handlers/sysinstance/launcher.go | 6 +- spinifex/lbagent/agent.go | 2 +- spinifex/network/host/imds_attach.go | 3 +- spinifex/network/host/imds_datapath.go | 7 +- spinifex/network/host/imds_datapath_test.go | 24 +- spinifex/services/viperblockd/viperblockd.go | 12 +- spinifex/types/ebs.go | 1 - spinifex/utils/images.go | 12 +- spinifex/vm/device_map.go | 2 - spinifex/vm/errors.go | 2 +- spinifex/vm/instance_cleaner.go | 4 +- spinifex/vm/lifecycle.go | 59 +- spinifex/vm/lifecycle_test.go | 14 +- spinifex/vm/mgmt_netcfg_test.go | 41 ++ spinifex/vm/network_helpers.go | 53 +- spinifex/vm/network_helpers_test.go | 103 ++-- spinifex/vm/network_plumber.go | 7 +- spinifex/vm/smbios_test.go | 75 +++ spinifex/vm/vm.go | 24 +- spinifex/vm/volumes.go | 2 +- spinifex/vm/volumes_test.go | 8 +- tests/e2e/harness/multinode_guest_ssh.go | 4 +- tests/e2e/harness/witness.go | 4 +- tests/e2e/iam/datapath_helpers_test.go | 2 +- tests/e2e/iam/imds_test.go | 96 ++- tests/e2e/multinode/baseline_test.go | 10 +- tests/e2e/multinode/guest_ssh_test.go | 24 +- tests/e2e/multinode/placement_nat_test.go | 10 +- tests/e2e/multinode/volume_durability_test.go | 4 +- tests/e2e/reboot/reboot_test.go | 2 +- tests/e2e/run-bm-multinode-e2e.sh | 22 +- tests/e2e/run-tofu-examples-e2e.sh | 12 +- tests/e2e/single/baseline_test.go | 10 +- tests/e2e/single/datapath_test.go | 2 +- tests/e2e/single/eip_test.go | 4 +- tests/e2e/single/eni_test.go | 2 +- tests/e2e/single/instance_test.go | 4 +- tests/e2e/single/lifecycle_test.go | 6 +- tests/e2e/single/natgw_test.go | 12 +- tests/e2e/single/securitygroup_test.go | 2 +- tests/e2e/single/volume_durability_test.go | 4 +- tests/e2e/single/vpc_test.go | 4 +- 81 files changed, 1050 insertions(+), 1496 deletions(-) create mode 100644 cmd/spinifex/cmd/admin_cabake_test.go create mode 100644 scripts/images/eks-node/mulga-mgmt-net.initd create mode 100644 scripts/images/eks-node/mulga-mgmt-net.sh delete mode 100644 spinifex/handlers/ec2/instance/helpers_test.go create mode 100644 spinifex/vm/mgmt_netcfg_test.go create mode 100644 spinifex/vm/smbios_test.go diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 2d7e5e7e2..46ee01b45 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -51,7 +51,7 @@ jobs: # --------------------------------------------------------------------------- single_provision: name: Single env · provision - runs-on: [ self-hosted, ci-single ] + runs-on: [self-hosted, ci-single] timeout-minutes: 40 outputs: wan_ip: ${{ steps.vm.outputs.wan_ip }} @@ -199,7 +199,7 @@ jobs: ssh "${SSH_OPTS[@]}" "$VM" \ "spx admin images import --file /tmp/eks-node-alpine.raw \ --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \ - --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null + --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1 ok "eks-node AMI imported" - name: Install kubectl on VM @@ -266,7 +266,7 @@ jobs: single_suite: name: Single env · suite needs: single_provision - runs-on: [ self-hosted, ci-single ] + runs-on: [self-hosted, ci-single] timeout-minutes: 45 strategy: fail-fast: false @@ -373,9 +373,9 @@ jobs: single_teardown: name: Single env · teardown - needs: [ single_provision, single_suite ] + needs: [single_provision, single_suite] if: always() - runs-on: [ self-hosted, ci-single ] + runs-on: [self-hosted, ci-single] timeout-minutes: 20 env: TOFU_LOG: /tmp/e2e-tofu-single-td-${{ github.run_id }}.log @@ -468,7 +468,7 @@ jobs: multi_provision: name: Multi env · provision if: ${{ inputs.run_multinode }} - runs-on: [ self-hosted, ci-multi ] + runs-on: [self-hosted, ci-multi] timeout-minutes: 45 outputs: node1_ip: ${{ steps.cluster.outputs.node1_ip }} @@ -615,12 +615,12 @@ jobs: multi_suite: name: Multi env · suite needs: multi_provision - runs-on: [ self-hosted, ci-multi ] + runs-on: [self-hosted, ci-multi] timeout-minutes: 60 strategy: fail-fast: false matrix: - suite: [ multinode ] + suite: [multinode] env: NODE1_IP: ${{ needs.multi_provision.outputs.node1_ip }} SSH_KEY: ${{ needs.multi_provision.outputs.ssh_key }} @@ -723,9 +723,9 @@ jobs: multi_teardown: name: Multi env · teardown - needs: [ multi_provision, multi_suite ] + needs: [multi_provision, multi_suite] if: always() && inputs.run_multinode - runs-on: [ self-hosted, ci-multi ] + runs-on: [self-hosted, ci-multi] timeout-minutes: 20 env: TOFU_LOG: /tmp/e2e-tofu-multi-td-${{ github.run_id }}.log @@ -786,7 +786,7 @@ jobs: e2e_suites_status: name: E2E suite blocks · summary if: always() - needs: [ single_suite, multi_suite ] + needs: [single_suite, multi_suite] runs-on: ubuntu-latest timeout-minutes: 5 steps: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c975b79a7..3d175924e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -224,13 +224,10 @@ jobs: scp "${SSH_OPTS[@]}" \ /tmp/eks-node-image-build/eks-node-alpine.raw \ "$VM":/tmp/eks-node-alpine.raw - # Drop the import progress bar (pterm redraws to stdout — thousands of - # lines over the SSH pipe). Errors go to stderr, so they survive; the - # exit code still gates the step. ssh "${SSH_OPTS[@]}" "$VM" \ "spx admin images import --file /tmp/eks-node-alpine.raw \ --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \ - --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null + --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1 ok "eks-node AMI imported" # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full diff --git a/.golangci.yml b/.golangci.yml index 046f86af9..b3a544936 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -57,7 +57,6 @@ linters: check-type-assertions: true exclude-functions: - (*github.com/nats-io/nats.go.Subscription).Unsubscribe # cleanup-only, errors not actionable - - (*github.com/kdomanski/iso9660.ImageWriter).Cleanup # deferred cleanup, errors not actionable govet: enable-all: true diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index b17b9b802..5ff0efb20 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -665,6 +665,13 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { EncryptionEnabled: mkey != nil, } + // Bake the deployment CA into the image trust store so a stock cloud image + // trusts the gateway from first boot. Best-effort: never fails the import. + // The CA path derives from the --spinifex-dir data root (its config/ symlink + // holds ca.pem), not NodeBaseDir(): node.BaseDir is unset in the import + // command, which would collapse the path to a relative config/ca.pem. + bakeCACertIntoImage(extractedImagePath, filepath.Join(baseDir, "config", "ca.pem")) + err = v_utils.ImportDiskImage(&s3Config, &vbConfig, extractedImagePath) if err != nil { @@ -677,6 +684,64 @@ func runimagesImportCmd(cmd *cobra.Command, args []string) { fmt.Printf("✅ Image import complete. Image-ID (AMI): %s\n", volumeId) } +// caBakeRunCommand installs the uploaded CA into the guest trust store across +// the distro families, then removes the staged copy. Each branch is gated on its +// updater existing so only the matching distro's branch writes anything (install +// -D creates the anchor dir on minimal images). Debian/Ubuntu use +// update-ca-certificates, RHEL/Rocky use update-ca-trust, and stock Alpine ships +// only ca-certificates-bundle (no updater, no anchor dir) so it falls back to +// appending the PEM to the static bundle. The install chain's exit code is +// preserved past the cleanup rm so a failure surfaces instead of a false success. +// The RHEL branch restorecon's only the trust-store paths it touched so the cert +// is correctly labelled without the image-wide --no-selinux-relabel (below) +// leaving an unlabelled bundle; restorecon is best-effort and never fails the bake. +const caBakeRunCommand = `( { command -v update-ca-certificates >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /usr/local/share/ca-certificates/spinifex.crt && update-ca-certificates; } || ` + + `{ command -v update-ca-trust >/dev/null && install -D -m644 /tmp/spinifex-ca.pem /etc/pki/ca-trust/source/anchors/spinifex.crt && update-ca-trust && ` + + `{ command -v restorecon >/dev/null 2>&1 && restorecon -RF /etc/pki/ca-trust/source/anchors/spinifex.crt /etc/pki/ca-trust/extracted >/dev/null 2>&1; true; }; } || ` + + `{ cat /tmp/spinifex-ca.pem >> /etc/ssl/certs/ca-certificates.crt; } ); ` + + `rc=$?; rm -f /tmp/spinifex-ca.pem; exit $rc` + +// caBakeTimeout bounds the virt-customize run so a stalled libguestfs appliance +// cannot hang the import indefinitely; on timeout the bake degrades to a skip. +const caBakeTimeout = 5 * time.Minute + +// caBakeCmd builds the virt-customize invocation that uploads the deployment CA +// into the disk image at imagePath and installs it into the guest trust store. +// --no-selinux-relabel stops virt-customize flagging the image for a first-boot +// SELinux autorelabel: that relabel+reboot corrupts XFS roots (RHEL/Rocky) on the +// reboot, so the run-command relabels only the touched trust paths instead. +func caBakeCmd(ctx context.Context, imagePath, caCertPath string) *exec.Cmd { + return exec.CommandContext(ctx, "virt-customize", "-a", imagePath, + "--no-selinux-relabel", + "--upload", caCertPath+":/tmp/spinifex-ca.pem", + "--run-command", caBakeRunCommand) +} + +// caBakeRunner resolves virt-customize and runs the CA bake; overridable in tests. +var caBakeRunner = func(imagePath, caCertPath string) ([]byte, error) { + if _, err := exec.LookPath("virt-customize"); err != nil { + return nil, fmt.Errorf("virt-customize not found: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), caBakeTimeout) + defer cancel() + return caBakeCmd(ctx, imagePath, caCertPath).CombinedOutput() +} + +// bakeCACertIntoImage uploads the deployment CA into the image's trust store via +// virt-customize so an imported stock image trusts the gateway from first boot. +// Best-effort: a missing CA, an absent virt-customize, or an image libguestfs +// cannot inspect logs and continues — image import never fails on the CA. +func bakeCACertIntoImage(imagePath, caCertPath string) { + if _, err := os.Stat(caCertPath); err != nil { + slog.Warn("CA bake skipped: deployment CA not found; imported image will not auto-trust the gateway", "ca", caCertPath, "err", err) + return + } + if out, err := caBakeRunner(imagePath, caCertPath); err != nil { + slog.Warn("CA bake skipped: virt-customize could not customize image; imported image will not auto-trust the gateway", "err", err, "output", string(out)) + return + } +} + func runimagesRemoveCmd(cmd *cobra.Command, args []string) { imageID, _ := cmd.Flags().GetString("image-id") force, _ := cmd.Flags().GetBool("force") diff --git a/cmd/spinifex/cmd/admin_cabake_test.go b/cmd/spinifex/cmd/admin_cabake_test.go new file mode 100644 index 000000000..1fb3a4a9d --- /dev/null +++ b/cmd/spinifex/cmd/admin_cabake_test.go @@ -0,0 +1,67 @@ +package cmd + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCABakeCmd_Args(t *testing.T) { + cmd := caBakeCmd(context.Background(), "/img/disk.raw", "/data/config/ca.pem") + args := cmd.Args + + assert.Equal(t, "virt-customize", filepath.Base(args[0])) + assert.Contains(t, args, "-a") + assert.Contains(t, args, "/img/disk.raw") + assert.Contains(t, args, "--upload") + assert.Contains(t, args, "/data/config/ca.pem:/tmp/spinifex-ca.pem") + assert.Contains(t, args, "--run-command") + + // --no-selinux-relabel stops virt-customize forcing a first-boot autorelabel, + // whose relabel+reboot corrupts XFS roots on RHEL/Rocky. + assert.Contains(t, args, "--no-selinux-relabel") + + // The run-command covers Debian/Ubuntu (update-ca-certificates), RHEL/Rocky + // (update-ca-trust), and the stock-Alpine fallback that appends to the static + // ca-certificates bundle when no updater is present. The RHEL branch relabels + // only the touched trust paths so the cert is labelled without a full relabel. + joined := strings.Join(args, " ") + assert.Contains(t, joined, "update-ca-certificates") + assert.Contains(t, joined, "update-ca-trust") + assert.Contains(t, joined, "restorecon") + assert.Contains(t, joined, "/etc/ssl/certs/ca-certificates.crt") +} + +func TestBakeCACertIntoImage_SkipsWhenCAMissing(t *testing.T) { + called := false + orig := caBakeRunner + caBakeRunner = func(string, string) ([]byte, error) { called = true; return nil, nil } + defer func() { caBakeRunner = orig }() + + // A missing CA must not invoke virt-customize and must not panic. + bakeCACertIntoImage("/img/disk.raw", filepath.Join(t.TempDir(), "absent-ca.pem")) + assert.False(t, called, "runner must not be called when the CA is absent") +} + +func TestBakeCACertIntoImage_ContinuesOnRunnerError(t *testing.T) { + caPath := filepath.Join(t.TempDir(), "ca.pem") + require.NoError(t, os.WriteFile(caPath, []byte("ca"), 0600)) + + called := false + orig := caBakeRunner + caBakeRunner = func(string, string) ([]byte, error) { + called = true + return []byte("libguestfs: cannot inspect image"), errors.New("exit status 1") + } + defer func() { caBakeRunner = orig }() + + // An image libguestfs cannot inspect must skip-and-continue, never panic/fail. + bakeCACertIntoImage("/img/disk.raw", caPath) + assert.True(t, called, "runner must be invoked when the CA is present") +} diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 3a07b7ab6..200c683fd 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -91,7 +91,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t | Command | Flags | Description | |---------|-------|-------------| -| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. | +| `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify` | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports skip checksum verification (operator-supplied media is outside Spinifex's trust boundary, the skip is logged at INFO for audit) and require an explicit `--boot-mode` because there is no catalog metadata to inherit from. `--skip-verify` bypasses verification for catalog imports and emits a WARN slog + stderr notice; use only for debugging or when upstream mirrors are confirmed-broken. On every import a best-effort `virt-customize` step bakes the deployment CA (`/config/ca.pem`) into the image's trust store before the block copy, so in-guest SDK-over-TLS calls to Spinifex endpoints trust the gateway from first boot; an image libguestfs cannot customize is imported as-is (CA-free) and logs a skip. Because the CA is fixed into the image, rotating the cluster CA requires re-importing affected images. | | `spx admin images list` | — | Lists available OS images that can be imported or downloaded | | `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-/config.json` (the DescribeImages barrier) followed by the rest of `ami-/` and `snap-ami-/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. | diff --git a/docs/DESIGN.md b/docs/DESIGN.md index a69f88630..19000ee0b 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -123,7 +123,7 @@ Daemons (`spinifex/daemon/daemon.go`) subscribe to NATS topics and handle reques When a daemon handles `RunInstances`: 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager` -2. **Volume Generation**: Creates boot, cloud-init, and EFI volumes via Viperblock +2. **Volume Generation**: Creates boot and EFI volumes via Viperblock 3. **Volume Mount**: Sends `ebs.mount` request to Viperblock, receives NBD URI 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management @@ -217,7 +217,6 @@ type EBSRequest struct { VolType string // gp2, io1, etc. Boot bool // Boot volume flag EFI bool // EFI boot volume - CloudInit bool // Cloud-init volume NBDURI string // NBD URI returned from mount } ``` diff --git a/docs/compute/launching-instances/README.md b/docs/compute/launching-instances/README.md index 4d68b4af3..949a25007 100644 --- a/docs/compute/launching-instances/README.md +++ b/docs/compute/launching-instances/README.md @@ -137,7 +137,7 @@ In development mode, find the QEMU port forward and connect via localhost: ```bash ps auxw | grep $INSTANCE_ID -ssh -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 -p +ssh -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 -p ``` For production SSH via public IPs, see [Setting Up Your Cluster](/docs/setting-up-your-cluster). diff --git a/docs/getting-started/setting-up-your-cluster/README.md b/docs/getting-started/setting-up-your-cluster/README.md index a9d1edc44..e5dfc0b31 100644 --- a/docs/getting-started/setting-up-your-cluster/README.md +++ b/docs/getting-started/setting-up-your-cluster/README.md @@ -243,7 +243,7 @@ SSH directly to the instance's public IP: PUBLIC_IP=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID \ --query 'Reservations[0].Instances[0].PublicIpAddress' --output text) -ssh -i ~/.ssh/spinifex-key ec2-user@$PUBLIC_IP +ssh -i ~/.ssh/spinifex-key ubuntu@$PUBLIC_IP ``` > **Note:** cloud-init takes 30-60 seconds to configure the instance. If SSH is refused, wait and retry. diff --git a/docs/reference-architectures/SMCI-MIG-orchestration/README.md b/docs/reference-architectures/SMCI-MIG-orchestration/README.md index 8f59a0971..451d65db3 100644 --- a/docs/reference-architectures/SMCI-MIG-orchestration/README.md +++ b/docs/reference-architectures/SMCI-MIG-orchestration/README.md @@ -271,7 +271,7 @@ MIG is enabled inside each guest VM, not on the host. SSH into each VM and enabl ```bash # Repeat on each VM IP -ssh -i ~/.ssh/spinifex-key ec2-user@ +ssh -i ~/.ssh/spinifex-key ubuntu@ sudo nvidia-smi -mig 1 nvidia-smi | grep "MIG M." diff --git a/docs/terraform-workbooks/bastion-private-subnet/README.md b/docs/terraform-workbooks/bastion-private-subnet/README.md index 95c0a01e7..516d773e6 100644 --- a/docs/terraform-workbooks/bastion-private-subnet/README.md +++ b/docs/terraform-workbooks/bastion-private-subnet/README.md @@ -93,13 +93,13 @@ tofu apply SSH into the bastion: ```bash -ssh -i bastion-demo.pem ec2-user@ +ssh -i bastion-demo.pem ubuntu@ ``` From the bastion, SSH to the private instance. The key is pre-installed at `~/.ssh/bastion-demo.pem` via cloud-init: ```bash -ssh -i ~/.ssh/bastion-demo.pem ec2-user@ +ssh -i ~/.ssh/bastion-demo.pem ubuntu@ ``` ### Step 4. Verify Isolation @@ -126,8 +126,8 @@ tofu destroy The private instance has no public IP — it is only reachable from the bastion. SSH to the bastion first, then use the pre-installed key to hop: ```bash -ssh -i bastion-demo.pem ec2-user@ -ssh -i ~/.ssh/bastion-demo.pem ec2-user@ +ssh -i bastion-demo.pem ubuntu@ +ssh -i ~/.ssh/bastion-demo.pem ubuntu@ ``` If the key is missing on the bastion, check that cloud-init completed successfully: diff --git a/docs/terraform-workbooks/bastion-private-subnet/main.tf b/docs/terraform-workbooks/bastion-private-subnet/main.tf index af22b83b5..3afcf0d47 100644 --- a/docs/terraform-workbooks/bastion-private-subnet/main.tf +++ b/docs/terraform-workbooks/bastion-private-subnet/main.tf @@ -18,11 +18,11 @@ # # After apply: # # SSH to the bastion -# ssh -i bastion-demo.pem ec2-user@ +# ssh -i bastion-demo.pem ubuntu@ # # # From the bastion, SSH to the private instance # # (the key is pre-installed at ~/.ssh/bastion-demo.pem via cloud-init) -# ssh -i ~/.ssh/bastion-demo.pem ec2-user@ +# ssh -i ~/.ssh/bastion-demo.pem ubuntu@ terraform { required_version = ">= 1.6.0" @@ -293,12 +293,12 @@ resource "aws_instance" "bastion" { user_data_base64 = base64encode(<<-USERDATA #!/bin/bash set -euo pipefail - mkdir -p /home/ec2-user/.ssh - cat > /home/ec2-user/.ssh/bastion-demo.pem <<'KEY' + mkdir -p /home/ubuntu/.ssh + cat > /home/ubuntu/.ssh/bastion-demo.pem <<'KEY' ${tls_private_key.bastion.private_key_openssh} KEY - chmod 600 /home/ec2-user/.ssh/bastion-demo.pem - chown -R ec2-user:ec2-user /home/ec2-user/.ssh + chmod 600 /home/ubuntu/.ssh/bastion-demo.pem + chown -R ubuntu:ubuntu /home/ubuntu/.ssh USERDATA ) @@ -342,10 +342,10 @@ output "private_instance_ip" { output "ssh_to_bastion" { description = "SSH to the bastion host" - value = "ssh -i bastion-demo.pem ec2-user@${aws_instance.bastion.public_ip}" + value = "ssh -i bastion-demo.pem ubuntu@${aws_instance.bastion.public_ip}" } output "ssh_to_private_from_bastion" { description = "From the bastion, SSH to the private instance (key is pre-installed via cloud-init)" - value = "ssh -i ~/.ssh/bastion-demo.pem ec2-user@${aws_instance.private.private_ip}" + value = "ssh -i ~/.ssh/bastion-demo.pem ubuntu@${aws_instance.private.private_ip}" } diff --git a/docs/terraform-workbooks/nginx-alb/README.md b/docs/terraform-workbooks/nginx-alb/README.md index c948d26f2..dd15cc238 100644 --- a/docs/terraform-workbooks/nginx-alb/README.md +++ b/docs/terraform-workbooks/nginx-alb/README.md @@ -223,7 +223,7 @@ aws ec2 describe-nat-gateways --query 'NatGateways[].[NatGatewayId,State]' The Nginx instances have no public IP, so you can't SSH in directly from your host. If you need to inspect cloud-init logs, launch a small jump host in the same VPC or run commands via the Spinifex console, then: ```bash -ssh -i nginx-alb-demo.pem ec2-user@ +ssh -i nginx-alb-demo.pem ubuntu@ sudo journalctl -u cloud-init --no-pager sudo systemctl status nginx ``` diff --git a/docs/terraform-workbooks/nginx-webserver/README.md b/docs/terraform-workbooks/nginx-webserver/README.md index 7035329a0..9e84a7360 100644 --- a/docs/terraform-workbooks/nginx-webserver/README.md +++ b/docs/terraform-workbooks/nginx-webserver/README.md @@ -81,7 +81,7 @@ After apply completes, use the outputs to test: ```bash curl http:// -ssh -i nginx-demo.pem ec2-user@ +ssh -i nginx-demo.pem ubuntu@ ``` Open the `web_url` output in your browser to see the Nginx welcome page. @@ -124,7 +124,7 @@ aws ec2 describe-instances --profile spinifex SSH into the instance and check cloud-init logs: ```bash -ssh -i nginx-demo.pem ec2-user@ +ssh -i nginx-demo.pem ubuntu@ sudo journalctl -u cloud-init --no-pager sudo systemctl status nginx ``` diff --git a/docs/terraform-workbooks/nginx-webserver/main.tf b/docs/terraform-workbooks/nginx-webserver/main.tf index 34d8b1f24..d1cb3e6b6 100644 --- a/docs/terraform-workbooks/nginx-webserver/main.tf +++ b/docs/terraform-workbooks/nginx-webserver/main.tf @@ -11,7 +11,7 @@ # # After apply: # curl http:// # Nginx welcome page -# ssh -i nginx-demo.pem ec2-user@ +# ssh -i nginx-demo.pem ubuntu@ terraform { required_version = ">= 1.6.0" @@ -283,7 +283,7 @@ output "public_ip" { } output "ssh_command" { - value = "ssh -i nginx-demo.pem ec2-user@${aws_instance.nginx.public_ip}" + value = "ssh -i nginx-demo.pem ubuntu@${aws_instance.nginx.public_ip}" } output "web_url" { diff --git a/docs/terraform-workbooks/s3-webapp/README.md b/docs/terraform-workbooks/s3-webapp/README.md index c8165b18f..53c47a167 100644 --- a/docs/terraform-workbooks/s3-webapp/README.md +++ b/docs/terraform-workbooks/s3-webapp/README.md @@ -143,7 +143,7 @@ aws s3 ls --profile spinifex --endpoint-url https://localhost:8443 SSH into the instance and check the service: ```bash -ssh -i s3-webapp-demo.pem ec2-user@ +ssh -i s3-webapp-demo.pem ubuntu@ sudo systemctl status s3-webapp sudo journalctl -u s3-webapp --no-pager -n 50 ``` @@ -153,7 +153,7 @@ sudo journalctl -u s3-webapp --no-pager -n 50 The instance reached S3 but was not authorized. The `.env` file deliberately holds **no** keys — credentials come from IMDS: ```bash -ssh -i s3-webapp-demo.pem ec2-user@ +ssh -i s3-webapp-demo.pem ubuntu@ cat /opt/webapp/.env # S3_ENDPOINT / S3_BUCKET / S3_REGION only — no keys ``` @@ -167,7 +167,7 @@ Check that: boto3 could not obtain credentials, which means the IMDS credential chain did not resolve. From the instance: ```bash -ssh -i s3-webapp-demo.pem ec2-user@ +ssh -i s3-webapp-demo.pem ubuntu@ # IMDSv2: fetch a token, then the role name behind the instance profile TOKEN=$(curl -sX PUT http://169.254.169.254/latest/api/token \ diff --git a/docs/terraform-workbooks/s3-webapp/main.tf b/docs/terraform-workbooks/s3-webapp/main.tf index 30e70b1be..95309f733 100644 --- a/docs/terraform-workbooks/s3-webapp/main.tf +++ b/docs/terraform-workbooks/s3-webapp/main.tf @@ -38,7 +38,7 @@ # # After apply: # curl http:// # File browser UI -# ssh -i s3-webapp-demo.pem ec2-user@ +# ssh -i s3-webapp-demo.pem ubuntu@ terraform { required_version = ">= 1.6.0" @@ -504,7 +504,7 @@ output "bucket_name" { } output "ssh_command" { - value = "ssh -i s3-webapp-demo.pem ec2-user@${aws_instance.webapp.public_ip}" + value = "ssh -i s3-webapp-demo.pem ubuntu@${aws_instance.webapp.public_ip}" } output "web_url" { diff --git a/go.mod b/go.mod index 5f643e051..e1a1ed16a 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,12 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 github.com/insomniacslk/dhcp v0.0.0-20260407060928-11b94ed970f2 - github.com/kdomanski/iso9660 v0.4.0 github.com/klauspost/cpuid/v2 v2.3.0 github.com/mulgadc/predastore v1.9.0 github.com/mulgadc/viperblock v1.9.0 github.com/nats-io/nats-server/v2 v2.14.2 github.com/nats-io/nats.go v1.52.0 + github.com/opencontainers/runtime-spec v1.3.0 github.com/ovn-kubernetes/libovsdb v0.8.1 github.com/pelletier/go-toml/v2 v2.4.0 github.com/pterm/pterm v0.12.83 @@ -130,7 +130,6 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/opencontainers/runtime-spec v1.3.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/go.sum b/go.sum index 33a1e61b1..e27908b10 100644 --- a/go.sum +++ b/go.sum @@ -1170,8 +1170,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= -github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/scripts/demo/deploy.sh b/scripts/demo/deploy.sh index 6de61f34d..1c84c284c 100755 --- a/scripts/demo/deploy.sh +++ b/scripts/demo/deploy.sh @@ -15,7 +15,7 @@ # # Optional: # SSH_KEY (default: ~/.ssh/spinifex-key) -# SSH_USER (default: ec2-user) +# SSH_USER (default: ubuntu) set -euo pipefail @@ -26,7 +26,7 @@ QWEN_IP="${QWEN_IP:?set QWEN_IP}" VIDEO_PATH="${VIDEO_PATH:?set VIDEO_PATH}" SSH_KEY="${SSH_KEY:-$HOME/.ssh/spinifex-key}" -SSH_USER="${SSH_USER:-ec2-user}" +SSH_USER="${SSH_USER:-ubuntu}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/scripts/images/eks-node/manifest.conf b/scripts/images/eks-node/manifest.conf index fd2dc4e33..82497fc11 100644 --- a/scripts/images/eks-node/manifest.conf +++ b/scripts/images/eks-node/manifest.conf @@ -40,6 +40,8 @@ scripts/images/eks-node/mulga-ebs-byid.initd:/etc/init.d/mulga-ebs-byid \ scripts/images/eks-node/mulga-ebs-byid.sh:/usr/local/sbin/mulga-ebs-byid \ scripts/images/eks-node/mulga-eks-provider-id.initd:/etc/init.d/mulga-eks-provider-id \ scripts/images/eks-node/mulga-eks-provider-id.sh:/usr/local/sbin/mulga-eks-provider-id \ +scripts/images/eks-node/mulga-mgmt-net.initd:/etc/init.d/mulga-mgmt-net \ +scripts/images/eks-node/mulga-mgmt-net.sh:/usr/local/sbin/mulga-mgmt-net \ scripts/images/eks-node/addons/spinifex-noop/0.1.0/noop.yaml:/usr/share/spinifex-eks/addons/spinifex-noop/0.1.0/noop.yaml \ scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml:/usr/share/spinifex-eks/addons/argocd/3.0.23/argocd.yaml \ scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml \ @@ -52,10 +54,14 @@ scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml:/u scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \ scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot" -# Only crond (etcd snapshot) + the role selector at bake. The selector adds the -# role-specific services (k3s/eks-token-webhook/k3s-first-boot/state-report or -# k3s-agent) to the default runlevel on first boot. -ENABLE_SERVICES="crond eks-node-role mulga-ebs-byid mulga-eks-provider-id" +# crond (etcd snapshot), the role selector, and the role-agnostic node services +# (EBS by-id, provider-id, mgmt NIC) at bake. mulga-mgmt-net configures the +# system-VM NICs from fw_cfg before cloud-init/k3s — DHCP on the primary data +# NIC so the Ec2 datasource reaches IMDS, static on mgmt0 (no-op without a +# netcfg blob). The selector adds the role-specific services +# (k3s/eks-token-webhook/k3s-first-boot/state-report or k3s-agent) to the +# default runlevel on first boot. +ENABLE_SERVICES="crond eks-node-role mulga-ebs-byid mulga-eks-provider-id mulga-mgmt-net" SETUP_SCRIPT="scripts/images/eks-node/setup.sh" diff --git a/scripts/images/eks-node/mulga-mgmt-net.initd b/scripts/images/eks-node/mulga-mgmt-net.initd new file mode 100644 index 000000000..5d15f2af4 --- /dev/null +++ b/scripts/images/eks-node/mulga-mgmt-net.initd @@ -0,0 +1,19 @@ +#!/sbin/openrc-run +# mulga-mgmt-net — boot oneshot that configures a multi-NIC system VM's NICs from +# the fw_cfg netcfg blob before cloud-init's network stage. BootAMI system VMs +# (EKS control plane) are dual-NIC: the data ENI is DHCP'd here so the Ec2 IMDS +# datasource can reach 169.254.169.254 (cloud-init can't pick the right NIC out +# of two on Alpine), and mgmt0's static br-mgmt address is applied. Role-agnostic +# and enabled on every node — a no-op when no netcfg blob is present. + +description="Configure system-VM NICs from fw_cfg netcfg (data DHCP + mgmt static)" + +depend() { + before cloud-init k3s k3s-agent +} + +start() { + ebegin "Configuring system-VM NICs from fw_cfg netcfg" + /usr/local/sbin/mulga-mgmt-net + eend $? +} diff --git a/scripts/images/eks-node/mulga-mgmt-net.sh b/scripts/images/eks-node/mulga-mgmt-net.sh new file mode 100644 index 000000000..8e521d13b --- /dev/null +++ b/scripts/images/eks-node/mulga-mgmt-net.sh @@ -0,0 +1,95 @@ +#!/bin/sh +# mulga-mgmt-net — boot oneshot that brings a multi-NIC BootAMI system VM's NICs +# up from the QEMU fw_cfg netcfg blob (opt/spinifex/netcfg) the host writes per +# launch. The EKS control plane is the case: cloud-init on a stock Alpine guest +# cannot reliably pick the right NIC out of two, so the host enumerates them and +# this oneshot configures each by MAC, before cloud-init's network stage: +# - NIC_DHCP=1: the primary data ENI. OVN serves DHCP; we lease it (one-shot) +# so the Ec2 IMDS datasource can reach 169.254.169.254, and pin a /32 to it so +# a link-local 169.254.0.0/16 route on another NIC cannot hijack IMDS. +# - NIC_CIDR: a static NIC (mgmt0 on br-mgmt, which has no DHCP). Applied +# address-only; NIC_DEFAULT is 0, so mgmt0 is never the default route. +# +# The blob is shell KEY=value, matching daemon.buildNetcfgBlob and +# build/microvm/init.sh; interfaces are matched by MAC. No-op when the blob is +# absent (a single-NIC agent/worker brings its one NIC up via cloud-init/IMDS), +# so the same image boots unchanged without one. +set -eu + +NETCFG="${MULGA_NETCFG:-/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw}" +IMDS_IP="169.254.169.254" + +# qemu_fw_cfg is a module on the stock Alpine cloud image; load it (best-effort) +# before reading. Skipped harmlessly when built-in or already loaded. +modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true + +if [ ! -f "$NETCFG" ]; then + echo "[mulga-mgmt-net] no fw_cfg netcfg; skipping system NIC setup" + exit 0 +fi + +# shellcheck disable=SC1090 +. "$NETCFG" + +# One-shot DHCP: configure the link then exit, leaving no daemon to fight +# cloud-init's later management of the primary ENI. Prefer busybox udhcpc; fall +# back to dhcpcd where udhcpc's config script is absent. +dhcp_oneshot() { + iface="$1" + if command -v udhcpc >/dev/null 2>&1 && [ -x /usr/share/udhcpc/default.script ]; then + udhcpc -i "$iface" -f -q -n -t 8 -T 2 >/dev/null 2>&1 + elif command -v dhcpcd >/dev/null 2>&1; then + dhcpcd -q -t 20 "$iface" >/dev/null 2>&1 + else + echo "[mulga-mgmt-net] no DHCP client (udhcpc/dhcpcd) for $iface" >&2 + return 1 + fi +} + +for n in 0 1 2 3 4 5; do + # ${VAR:-} keeps the unset NIC_* slots safe under `set -u`. + eval "mac=\${NIC${n}_MAC:-}" + [ -z "$mac" ] && continue + + iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :) + if [ -z "$iface" ]; then + echo "[mulga-mgmt-net] WARNING: no interface for MAC $mac (NIC$n); skipping" >&2 + continue + fi + + eval "dhcp=\${NIC${n}_DHCP:-}" + eval "cidr=\${NIC${n}_CIDR:-}" + ip link set "$iface" up + + if [ "$dhcp" = "1" ]; then + if dhcp_oneshot "$iface"; then + echo "[mulga-mgmt-net] data NIC $iface ($mac) up via DHCP" + else + echo "[mulga-mgmt-net] ERROR: no DHCP lease on data NIC $iface ($mac)" >&2 + fi + # Pin IMDS to the data NIC so a link-local 169.254.0.0/16 route on another + # interface cannot steal the metadata path. Route via the gateway, not + # on-link: the host demuxes IMDS sent to the gateway MAC, never ARP-answers .254. + gw=$(ip -4 route show default dev "$iface" 2>/dev/null | awk '{print $3; exit}') + if [ -n "$gw" ]; then + ip route replace "${IMDS_IP}/32" via "$gw" dev "$iface" 2>/dev/null || true + else + ip route replace "${IMDS_IP}/32" dev "$iface" 2>/dev/null || true + fi + continue + fi + + if [ -z "$cidr" ]; then + echo "[mulga-mgmt-net] brought up $iface ($mac), no CIDR" + continue + fi + # `replace` is idempotent — adds the address if absent, no-op on a stop/start + # re-attach that reuses a surviving interface. A real failure must surface, + # not strand mgmt0 IP-less. + if ip addr replace "$cidr" dev "$iface"; then + echo "[mulga-mgmt-net] configured $iface ($mac) with $cidr" + else + echo "[mulga-mgmt-net] ERROR: failed to set $cidr on $iface ($mac)" >&2 + exit 1 + fi +done diff --git a/scripts/images/eks-node/setup.sh b/scripts/images/eks-node/setup.sh index 782b8febd..135f23f1b 100755 --- a/scripts/images/eks-node/setup.sh +++ b/scripts/images/eks-node/setup.sh @@ -49,6 +49,7 @@ chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \ /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync chmod 0755 /etc/init.d/mulga-ebs-byid /usr/local/sbin/mulga-ebs-byid chmod 0755 /etc/init.d/mulga-eks-provider-id /usr/local/sbin/mulga-eks-provider-id +chmod 0755 /etc/init.d/mulga-mgmt-net /usr/local/sbin/mulga-mgmt-net chmod 0755 /etc/periodic/daily/mulga-eks-etcd-snapshot # EBS by-id bridge: route every virtio-blk event through mulga-ebs-byid, which diff --git a/scripts/launch-gpu.sh b/scripts/launch-gpu.sh index 7b89a3196..d9212f114 100755 --- a/scripts/launch-gpu.sh +++ b/scripts/launch-gpu.sh @@ -6,7 +6,7 @@ export AWS_PROFILE=spinifex # Resolve the actual user's home — $HOME is /root when run under sudo. _USER_HOME=$(getent passwd "${SUDO_USER:-$(id -un)}" | cut -d: -f6) SSH_KEY="${SSH_KEY:-${_USER_HOME}/.ssh/spinifex-key}" -SSH_USER="${SSH_USER:-ec2-user}" +SSH_USER="${SSH_USER:-ubuntu}" SSH_TIMEOUT="${SSH_TIMEOUT:-300}" usage() { @@ -28,7 +28,7 @@ Options: Env overrides: SSH_KEY Path to SSH private key (default: ~/.ssh/spinifex-key) - SSH_USER SSH user inside guest (default: ec2-user) + SSH_USER SSH user inside guest (default: ubuntu) SSH_TIMEOUT Seconds to wait for initial SSH (default: 300) EOF } diff --git a/scripts/run-bench.sh b/scripts/run-bench.sh index c8b10e9a0..94caa1141 100755 --- a/scripts/run-bench.sh +++ b/scripts/run-bench.sh @@ -137,7 +137,7 @@ SSH_READY=false while [ $SSH_ATTEMPTS -lt $MAX_SSH_ATTEMPTS ]; do ssh-keyscan -p "$SSH_PORT" 127.0.0.1 >> ~/.ssh/known_hosts 2>/dev/null - if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p "$SSH_PORT" -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 'echo ready' >/dev/null 2>&1; then + if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p "$SSH_PORT" -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 'echo ready' >/dev/null 2>&1; then SSH_READY=true echo "SSH is ready!" break @@ -156,7 +156,7 @@ if [ "$SSH_READY" != "true" ]; then fi echo "Copying benchmark script to instance..." -scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$SSH_PORT" -i ~/.ssh/spinifex-key "$BENCH_SCRIPT" ec2-user@127.0.0.1:~/disk-performance.sh +scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$SSH_PORT" -i ~/.ssh/spinifex-key "$BENCH_SCRIPT" ubuntu@127.0.0.1:~/disk-performance.sh if [ $? -ne 0 ]; then echo "Error: Failed to copy benchmark script to instance" @@ -165,7 +165,7 @@ fi # Execute benchmark script echo "Running benchmark on instance..." -ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p "$SSH_PORT" -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 'chmod +x ~/disk-performance.sh && ~/disk-performance.sh' > /tmp/spinifex-vm-disk.log 2>&1 +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p "$SSH_PORT" -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 'chmod +x ~/disk-performance.sh && ~/disk-performance.sh' > /tmp/spinifex-vm-disk.log 2>&1 if [ $? -ne 0 ]; then echo "Error: Benchmark execution failed, check /tmp/spinifex-vm-disk.log for details" diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 83e1bb8c1..8b8543436 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -226,7 +226,7 @@ for _i in $(seq 1 300); do -o BatchMode=yes \ -p "$SSH_INST_PORT" \ -i "$SSH_KEY" \ - ec2-user@"$SSH_INST_HOST" 'echo ready' >/dev/null 2>&1; then + ubuntu@"$SSH_INST_HOST" 'echo ready' >/dev/null 2>&1; then SSH_READY=1 break fi @@ -246,10 +246,10 @@ SSH_OUT=$(ssh -o StrictHostKeyChecking=no \ -o BatchMode=yes \ -p "$SSH_INST_PORT" \ -i "$SSH_KEY" \ - ec2-user@"$SSH_INST_HOST" 'id && hostname' 2>&1) + ubuntu@"$SSH_INST_HOST" 'id && hostname' 2>&1) echo " $SSH_OUT" -if ! echo "$SSH_OUT" | grep -q "ec2-user"; then - echo "❌ Expected ec2-user in SSH output" +if ! echo "$SSH_OUT" | grep -q "ubuntu"; then + echo "❌ Expected ubuntu in SSH output" exit 1 fi @@ -261,7 +261,7 @@ if [[ "$TEST_GPU" == "1" ]]; then if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ -o ConnectTimeout=5 -o BatchMode=yes \ -p "$SSH_INST_PORT" -i "$SSH_KEY" \ - ec2-user@"$SSH_INST_HOST" \ + ubuntu@"$SSH_INST_HOST" \ 'lspci 2>/dev/null | grep -iE "nvidia|amdgpu|instinct"' 2>/dev/null; then GPU_VISIBLE=1 break diff --git a/spinifex/daemon/daemon_system_instance.go b/spinifex/daemon/daemon_system_instance.go index 179d877fc..c4151ab65 100644 --- a/spinifex/daemon/daemon_system_instance.go +++ b/spinifex/daemon/daemon_system_instance.go @@ -383,16 +383,17 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) } // launchAMISystemInstance is the BootAMI branch of LaunchSystemInstance: a full -// AMI boot (root volume cloned from an AMI, cloud-init user-data + network -// config seed) for a system-managed VM, with a management-bridge NIC so the +// AMI boot (root volume cloned from an AMI, bootstrapped from the Ec2 IMDS +// datasource) for a system-managed VM, with a management-bridge NIC so the // guest can reach the daemon (NATS/AWSGW) off its tenant VPC subnet. // // It mirrors the daemon RunInstances handler's Prepare → Insert → Launch split, // allocating the mgmt NIC on the VM record between Insert and Launch so the -// cloud-init network-config rendered during Launch carries a static mgmt0 -// interface. The instance is owned by input.AccountID (the account its -// pre-created ENI lives in) and tagged input.ManagedBy so customer listings -// hide it. +// fw_cfg netcfg blob built during Launch enumerates both the primary data ENI +// (DHCP) and the static mgmt0 address — a multi-NIC Alpine guest cannot pick the +// right NIC for the Ec2 datasource on its own. The instance is owned by +// input.AccountID (the account its pre-created ENI lives in) and tagged +// input.ManagedBy so customer listings hide it. func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput) (*sysinstance.SystemInstanceOutput, error) { if d.instanceService == nil { return nil, errors.New("sysinstance: instance service not initialized") @@ -441,8 +442,8 @@ func (d *Daemon) launchAMISystemInstance(input *sysinstance.SystemInstanceInput) d.vmMgr.Insert(instance) } - // Management NIC must be set before LaunchRunInstances so the cloud-init - // network-config (built during launch) renders a static mgmt0 interface. + // Management NIC must be set before LaunchRunInstances so the fw_cfg netcfg + // blob (built during launch) carries the static mgmt0 address. if err := d.attachSystemMgmtNIC(inst); err != nil { d.vmMgr.MarkFailed(inst, "mgmt_nic_setup_failed") return nil, err diff --git a/spinifex/daemon/daemon_test.go b/spinifex/daemon/daemon_test.go index 14dfaad72..5aa3236f4 100644 --- a/spinifex/daemon/daemon_test.go +++ b/spinifex/daemon/daemon_test.go @@ -1619,7 +1619,7 @@ func TestResourceManager_ConcurrentAccess(t *testing.T) { // TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination tests that // the instanceCleanerAdapter correctly handles DeleteOnTermination for each -// volume type: internal volumes (EFI, cloud-init) always go through +// volume type: internal volumes (EFI) always go through // ebs.delete; user volumes only when DeleteOnTermination is true. func TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination(t *testing.T) { natsURL := sharedNATSURL @@ -1628,7 +1628,7 @@ func TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination(t *testing.T) var mu sync.Mutex ebsDeletedVolumes := make(map[string]bool) - const expectedDeletes = 2 // EFI + cloud-init; vol-root has no S3 backend + const expectedDeletes = 1 // EFI; vol-root has no S3 backend allDeletes := make(chan struct{}) deleteSub, err := daemon.natsConn.Subscribe("ebs.delete", func(msg *nats.Msg) { @@ -1655,7 +1655,6 @@ func TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination(t *testing.T) Requests: []types.EBSRequest{ {Name: "vol-root", Boot: true, DeleteOnTermination: true}, {Name: "vol-root-efi", EFI: true}, - {Name: "vol-root-cloudinit", CloudInit: true}, }, }, } @@ -1672,9 +1671,8 @@ func TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination(t *testing.T) mu.Lock() defer mu.Unlock() - // Internal volumes (EFI, cloud-init) should receive ebs.delete + // Internal volumes (EFI) should receive ebs.delete assert.True(t, ebsDeletedVolumes["vol-root-efi"], "EFI volume should receive ebs.delete") - assert.True(t, ebsDeletedVolumes["vol-root-cloudinit"], "Cloud-init volume should receive ebs.delete") } // TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination_False verifies @@ -1687,7 +1685,7 @@ func TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination_False(t *testi var mu sync.Mutex ebsDeletedVolumes := make(map[string]bool) - const expectedDeletes = 2 // only the two internal volumes; vol-keep is skipped + const expectedDeletes = 1 // only the EFI internal volume; vol-keep is skipped allDeletes := make(chan struct{}) deleteSub, err := daemon.natsConn.Subscribe("ebs.delete", func(msg *nats.Msg) { @@ -1714,7 +1712,6 @@ func TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination_False(t *testi Requests: []types.EBSRequest{ {Name: "vol-keep", Boot: true, DeleteOnTermination: false}, {Name: "vol-keep-efi", EFI: true}, - {Name: "vol-keep-cloudinit", CloudInit: true}, }, }, } @@ -1733,7 +1730,6 @@ func TestInstanceCleanerAdapter_DeleteVolumes_DeleteOnTermination_False(t *testi // Internal volumes still get ebs.delete (always cleaned up) assert.True(t, ebsDeletedVolumes["vol-keep-efi"], "EFI volume should receive ebs.delete even when root has DeleteOnTermination=false") - assert.True(t, ebsDeletedVolumes["vol-keep-cloudinit"], "Cloud-init volume should receive ebs.delete even when root has DeleteOnTermination=false") // Root volume with DeleteOnTermination=false should NOT receive ebs.delete assert.False(t, ebsDeletedVolumes["vol-keep"], "Root volume with DeleteOnTermination=false should NOT be deleted") @@ -2024,40 +2020,6 @@ func TestHandleEC2Events_DetachVolume(t *testing.T) { instance.EBSRequests.Mu.Unlock() }) - t.Run("CloudInitVolumeProtection", func(t *testing.T) { - ciVolumeID := "vol-cloudinit-protected" - - instance.EBSRequests.Mu.Lock() - instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, types.EBSRequest{ - Name: ciVolumeID, - CloudInit: true, - }) - instance.EBSRequests.Mu.Unlock() - - command := types.EC2InstanceCommand{ - ID: instanceID, - Attributes: types.EC2CommandAttributes{ - DetachVolume: true, - }, - DetachVolumeData: &types.DetachVolumeData{ - VolumeID: ciVolumeID, - }, - } - data, _ := json.Marshal(command) - - resp, err := natsRequest(daemon.natsConn, - fmt.Sprintf("ec2.cmd.%s", instanceID), - data, - 5*time.Second, - ) - require.NoError(t, err) - assert.Contains(t, string(resp.Data), "OperationNotPermitted") - - instance.EBSRequests.Mu.Lock() - instance.EBSRequests.Requests = instance.EBSRequests.Requests[:len(instance.EBSRequests.Requests)-1] - instance.EBSRequests.Mu.Unlock() - }) - t.Run("DeviceMismatch", func(t *testing.T) { command := types.EC2InstanceCommand{ ID: instanceID, diff --git a/spinifex/daemon/resource.go b/spinifex/daemon/resource.go index 140121cc9..a7aa27848 100644 --- a/spinifex/daemon/resource.go +++ b/spinifex/daemon/resource.go @@ -185,10 +185,10 @@ const ( defaultNbdkitAuxMiB = 96 ) -// Default volume layout: one main root volume + two aux volumes (cloud-init, efi). +// Default volume layout: one main root volume + one aux volume (efi). const ( defaultMainVolumes = 1 - defaultAuxVolumes = 2 + defaultAuxVolumes = 1 ) // resolveNbdkitCharge returns per-volume nbdkit charges, overridable via env. diff --git a/spinifex/daemon/vm_adapters.go b/spinifex/daemon/vm_adapters.go index 3b9c90540..60a55e041 100644 --- a/spinifex/daemon/vm_adapters.go +++ b/spinifex/daemon/vm_adapters.go @@ -177,7 +177,7 @@ func (a *volumeMounterAdapter) Unmount(instance *vm.VM) error { "instance", instance.ID, "volume", ebsRequest.Name, "data", string(msg.Data)) } - if sealed && !ebsRequest.EFI && !ebsRequest.CloudInit && a.volState != nil { + if sealed && !ebsRequest.EFI && a.volState != nil { if err := a.volState.UpdateVolumeState(ebsRequest.Name, "available", "", ""); err != nil { slog.Error("Failed to update volume state to available after unmount", "volumeId", ebsRequest.Name, "err", err) @@ -561,8 +561,8 @@ func newInstanceCleanerAdapter(d *Daemon) *instanceCleanerAdapter { return &instanceCleanerAdapter{d: d} } -// DeleteVolumes deletes EFI / cloud-init internal volumes via ebs.delete -// and user volumes flagged DeleteOnTermination via the volume service. +// DeleteVolumes deletes EFI internal volumes via ebs.delete and user volumes +// flagged DeleteOnTermination via the volume service. // Errors are logged per volume; partial failure is tolerated. func (a *instanceCleanerAdapter) DeleteVolumes(instance *vm.VM) error { instance.EBSRequests.Mu.Lock() @@ -570,11 +570,10 @@ func (a *instanceCleanerAdapter) DeleteVolumes(instance *vm.VM) error { var firstErr error for _, ebsRequest := range instance.EBSRequests.Requests { - // Internal volumes (EFI, cloud-init) always go through ebs.delete to - // stop their viperblockd processes. S3 data is cleaned up via the - // parent root volume's DeleteVolume (which removes -efi/ and - // -cloudinit/ prefixes). - if ebsRequest.EFI || ebsRequest.CloudInit { + // Internal volumes (EFI) always go through ebs.delete to stop their + // viperblockd processes. S3 data is cleaned up via the parent root + // volume's DeleteVolume (which removes the -efi/ prefix). + if ebsRequest.EFI { ebsDeleteData, err := json.Marshal(types.EBSDeleteRequest{Volume: ebsRequest.Name}) if err != nil { slog.Error("Failed to marshal ebs.delete request for internal volume", diff --git a/spinifex/handlers/ec2/instance/helpers_test.go b/spinifex/handlers/ec2/instance/helpers_test.go deleted file mode 100644 index e1e38f26e..000000000 --- a/spinifex/handlers/ec2/instance/helpers_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package handlers_ec2_instance - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// --- generateNetworkConfig --- - -func TestGenerateNetworkConfig_BothEmpty(t *testing.T) { - cfg := generateNetworkConfig("", "", "", "", nil) - assert.Equal(t, cloudInitNetworkConfigWildcard, cfg) -} - -func TestGenerateNetworkConfig_OneEmpty(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "", "", nil) - assert.Contains(t, cfg, "vpc0:", "eniMAC alone should produce per-interface config") - assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC") - - cfg = generateNetworkConfig("", "02:00:00:dd:ee:ff", "", "", nil) - assert.Equal(t, cloudInitNetworkConfigWildcard, cfg, "should fall back to wildcard if eniMAC empty") -} - -func TestGenerateNetworkConfig_DualNIC(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil) - assert.Contains(t, cfg, "version: 2") - assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`) - assert.Contains(t, cfg, `macaddress: "02:00:00:dd:ee:ff"`) - assert.Contains(t, cfg, "use-routes: false") - assert.Contains(t, cfg, "use-dns: false") - assert.Contains(t, cfg, "vpc0:") - assert.Contains(t, cfg, "dev0:") - assert.NotContains(t, cfg, "mgmt0:") -} - -func TestGenerateNetworkConfig_TripleNIC(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "10.15.8.101", nil) - assert.Contains(t, cfg, "version: 2") - assert.Contains(t, cfg, `macaddress: "02:00:00:aa:bb:cc"`) - assert.Contains(t, cfg, `macaddress: "02:de:00:dd:ee:ff"`) - assert.Contains(t, cfg, "mgmt0:") - assert.Contains(t, cfg, `macaddress: "02:a0:00:11:22:33"`) - assert.Contains(t, cfg, `"10.15.8.101/24"`) - // mgmt NIC should not have DHCP or routes - assert.Contains(t, cfg, "vpc0:") - assert.Contains(t, cfg, "dev0:") -} - -func TestGenerateNetworkConfig_MgmtWithoutDev(t *testing.T) { - // System instances: eniMAC + mgmtMAC, no devMAC — should get per-interface config with mgmt NIC - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "", "02:a0:00:11:22:33", "10.15.8.101", nil) - assert.Contains(t, cfg, "vpc0:") - assert.NotContains(t, cfg, "dev0:", "no dev NIC without devMAC") - assert.Contains(t, cfg, "mgmt0:") - assert.Contains(t, cfg, `macaddress: "02:a0:00:11:22:33"`) - assert.Contains(t, cfg, `"10.15.8.101/24"`) -} - -func TestGenerateNetworkConfig_MgmtMACWithoutIP(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "02:a0:00:11:22:33", "", nil) - assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without IP") -} - -func TestGenerateNetworkConfig_MgmtIPWithoutMAC(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:de:00:dd:ee:ff", "", "10.15.8.101", nil) - assert.NotContains(t, cfg, "mgmt0:", "mgmt NIC should not appear without MAC") -} - -// The per-tap IMDS datapath intercepts 169.254.169.254 at the host tap, so guests -// no longer need an in-guest on-link route — the network-config must not emit one -// on any NIC (primary, extra, or dev). -func TestGenerateNetworkConfig_NoIMDSRoute(t *testing.T) { - cfg := generateNetworkConfig("02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", - []string{"02:00:00:bb:bb:bb"}) - assert.Contains(t, cfg, "vpc0:") - assert.NotContains(t, cfg, "169.254.169.254", "no in-guest IMDS route (served at the tap)") -} - -// No family carries the in-guest IMDS route now that it is served at the tap. -// RHEL renders its NICs via NM keyfiles (config disabled here), so it is covered -// in TestBuildRHELCloudInit instead. -func TestSelectNetworkConfigForFamily_NoIMDSRoute(t *testing.T) { - for _, family := range []string{"debian", "ubuntu", "alpine"} { - cfg := selectNetworkConfigForFamily(family, "02:00:00:aa:bb:cc", "02:00:00:dd:ee:ff", "", "", nil) - assert.Contains(t, cfg, "vpc0:", family) - assert.NotContains(t, cfg, "169.254.169.254", family+" must not carry the IMDS route") - } -} - -// Route for multi-node LB mgmt traffic is handled via the fw_cfg netcfg -// blob delivered to the LB microVM, not in the network-config. diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index 6bfc06498..178c6d0db 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -1,24 +1,18 @@ package handlers_ec2_instance import ( - "bytes" "encoding/base64" "encoding/json" "errors" "fmt" - "io" "log/slog" "os" - "path/filepath" "strconv" "strings" - "text/template" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - awss3 "github.com/aws/aws-sdk-go/service/s3" - "github.com/kdomanski/iso9660" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" @@ -36,254 +30,6 @@ import ( "github.com/nats-io/nats.go" ) -const cloudInitUserDataTemplate = `#cloud-config -{{if .SSHKey}} -users: - - name: {{.Username}} - shell: /bin/sh -{{- if .AlpineUnlockPasswd}} - lock_passwd: false - hashed_passwd: "*" -{{- end}} - groups: - - {{.SudoGroup}} - sudo: "ALL=(ALL) NOPASSWD:ALL" - ssh_authorized_keys: - - {{.SSHKey}} -{{end}} -ssh_pwauth: false - -hostname: {{.Hostname}} -manage_etc_hosts: true - -{{if .CACertPEM}} -ca_certs: - trusted: - - | -{{.CACertPEM}} -{{end}} - -{{if .UserDataCloudConfig}} -{{.UserDataCloudConfig}} -{{end}} - -{{if or .RHELWriteFiles .UserDataScript}} -write_files: -{{- if .RHELWriteFiles}} -{{.RHELWriteFiles}}{{end}}{{if .UserDataScript}} - - path: /tmp/cloud-init-startup.sh - permissions: '0755' - content: | -{{.UserDataScript}}{{end}} -{{end}} -{{if or .RHELRunCmd .UserDataScript}} -runcmd: -{{- if .RHELRunCmd}} -{{.RHELRunCmd}}{{end}}{{if .UserDataScript}} - - [ "/bin/bash", "/tmp/cloud-init-startup.sh" ] -{{end}} -{{end}} -` - -const cloudInitMetaTemplate = `# meta-data -instance-id: {{.InstanceID}} -local-hostname: {{.Hostname}} -` - -// cloudInitNetworkConfigWildcard enables DHCP on all NICs via "e*" glob, -// covering both traditional (eth0) and predictable (ens3) names. -// Used when there is no dual-NIC setup (non-VPC or no DEV_NETWORKING). -const cloudInitNetworkConfigWildcard = `network: - version: 2 - ethernets: - allnics: - match: - name: "e*" - dhcp4: true - dhcp-identifier: mac -` - -// cloudInitNetworkConfigDisabled suppresses cloud-init network rendering. -// Used on RHEL guests where NM keyfiles are written via write_files; -// without this stub cloud-init would drop a competing autoconnect keyfile. -const cloudInitNetworkConfigDisabled = `network: - config: disabled -` - -// selectNetworkConfigForFamily returns the cloud-init network-config for the distro. -// RHEL gets "config: disabled" to prevent cloud-init from dropping a competing -// NM keyfile alongside the ones written via user-data write_files. -func selectNetworkConfigForFamily(distroFamily, eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) string { - if distroFamily == "rhel" { - return cloudInitNetworkConfigDisabled - } - return generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP, extraENIMACs) -} - -// generateNetworkConfig produces the cloud-init network-config for the instance. -// Emits per-interface blocks when eniMAC is set; falls back to wildcard DHCP -// when eniMAC is empty. -func generateNetworkConfig(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) string { - if eniMAC == "" { - return cloudInitNetworkConfigWildcard - } - cfg := fmt.Sprintf(`network: - version: 2 - ethernets: - vpc0: - match: - macaddress: "%s" - dhcp4: true - dhcp-identifier: mac -`, eniMAC) - - for i, mac := range extraENIMACs { - if mac == "" { - continue - } - cfg += fmt.Sprintf(` vpc%d: - match: - macaddress: "%s" - dhcp4: true - dhcp-identifier: mac -`, i+1, mac) - } - - if devMAC != "" { - cfg += fmt.Sprintf(` dev0: - match: - macaddress: "%s" - dhcp4: true - dhcp-identifier: mac - dhcp4-overrides: - use-routes: false - use-dns: false -`, devMAC) - } - - if mgmtMAC != "" && mgmtIP != "" { - cfg += fmt.Sprintf(` mgmt0: - match: - macaddress: "%s" - addresses: - - "%s/24" -`, mgmtMAC, mgmtIP) - // LB mgmt route is delivered via fw_cfg netcfg, not here. - } - - return cfg -} - -type CloudInitData struct { - Username string - SSHKey string - Hostname string - UserDataCloudConfig string - UserDataScript string - CACertPEM string - // DistroFamily selects per-distro rendering: "debian" | "rhel" | "alpine". - // Empty falls through to debian (legacy AMIs without this field). - DistroFamily string - // SudoGroup is "sudo" on debian/ubuntu/alpine, "wheel" on RHEL-family. - SudoGroup string - // RHELWriteFiles is the pre-indented YAML NM keyfile block (one per NIC). - // Empty on non-RHEL guests; those use the network-config ISO instead. - RHELWriteFiles string - // RHELRunCmd is the pre-indented runcmd YAML (restorecon + nmcli reload/up). - // Empty on non-RHEL guests. - RHELRunCmd string - // AlpineUnlockPasswd renders the user with a disabled-but-unlocked password - // (lock_passwd: false, hashed_passwd: "*"). Alpine's sshd is built without - // PAM and rejects any login for a locked ("!") account; "*" keeps password - // auth disabled while letting pubkey auth through. - AlpineUnlockPasswd bool -} - -// buildRHELCloudInit produces the cloud-init write_files (NM keyfiles) and -// runcmd blocks for RHEL networking. Returns empty strings when eniMAC is empty. -// File mode 0600 is required — NM ignores world-readable keyfiles. -func buildRHELCloudInit(eniMAC, devMAC, mgmtMAC, mgmtIP string, extraENIMACs []string) (writeFiles, runCmd string) { - if eniMAC == "" { - return "", "" - } - - var wf, rc strings.Builder - - rc.WriteString(" - [ restorecon, -R, /etc/NetworkManager/system-connections/ ]\n") - rc.WriteString(" - [ nmcli, connection, reload ]\n") - - appendRHELDHCPKeyfile(&wf, "vpc0", eniMAC, false) - rc.WriteString(" - [ nmcli, connection, up, vpc0 ]\n") - - for i, mac := range extraENIMACs { - if mac == "" { - continue - } - name := fmt.Sprintf("vpc%d", i+1) - appendRHELDHCPKeyfile(&wf, name, mac, false) - fmt.Fprintf(&rc, " - [ nmcli, connection, up, %s ]\n", name) - } - - if devMAC != "" { - appendRHELDHCPKeyfile(&wf, "dev0", devMAC, true) - rc.WriteString(" - [ nmcli, connection, up, dev0 ]\n") - } - - if mgmtMAC != "" && mgmtIP != "" { - appendRHELStaticKeyfile(&wf, "mgmt0", mgmtMAC, mgmtIP) - rc.WriteString(" - [ nmcli, connection, up, mgmt0 ]\n") - } - - // Trim trailing newline so the template's {{.RHELWriteFiles}} doesn't - // produce a blank line before the user-script write_files entry on - // instances that have both. - return strings.TrimRight(wf.String(), "\n"), strings.TrimRight(rc.String(), "\n") -} - -func appendRHELDHCPKeyfile(b *strings.Builder, name, mac string, suppressDefaults bool) { - fmt.Fprintf(b, " - path: /etc/NetworkManager/system-connections/%s.nmconnection\n", name) - b.WriteString(" owner: root:root\n") - b.WriteString(" permissions: '0600'\n") - b.WriteString(" content: |\n") - b.WriteString(" [connection]\n") - fmt.Fprintf(b, " id=%s\n", name) - b.WriteString(" type=ethernet\n") - b.WriteString(" [ethernet]\n") - fmt.Fprintf(b, " mac-address=%s\n", mac) - b.WriteString(" [ipv4]\n") - b.WriteString(" method=auto\n") - b.WriteString(" dhcp-client-id=mac\n") - if suppressDefaults { - // Dev NIC gets DHCP IP for hostfwd but must not install routes or DNS. - b.WriteString(" never-default=true\n") - b.WriteString(" ignore-auto-dns=true\n") - } - b.WriteString(" [ipv6]\n") - b.WriteString(" method=disabled\n") -} - -func appendRHELStaticKeyfile(b *strings.Builder, name, mac, ip string) { - fmt.Fprintf(b, " - path: /etc/NetworkManager/system-connections/%s.nmconnection\n", name) - b.WriteString(" owner: root:root\n") - b.WriteString(" permissions: '0600'\n") - b.WriteString(" content: |\n") - b.WriteString(" [connection]\n") - fmt.Fprintf(b, " id=%s\n", name) - b.WriteString(" type=ethernet\n") - b.WriteString(" [ethernet]\n") - fmt.Fprintf(b, " mac-address=%s\n", mac) - b.WriteString(" [ipv4]\n") - b.WriteString(" method=manual\n") - fmt.Fprintf(b, " addresses=%s/24\n", ip) - b.WriteString(" [ipv6]\n") - b.WriteString(" method=disabled\n") -} - -type CloudInitMetaData struct { - InstanceID string - Hostname string -} - // VolumeInfo holds volume information returned from GenerateVolumes // for populating BlockDeviceMappings in the EC2 API response type VolumeInfo struct { @@ -1444,15 +1190,7 @@ func (s *InstanceServiceImpl) GenerateVolumes(input *ec2.RunInstancesInput, inst } } - // Step 3: Create cloud-init volume if needed - if input.KeyName != nil && *input.KeyName != "" || (input.UserData != nil && *input.UserData != "") { - err = s.prepareCloudInitVolume(input, imageId, volumeConfig, instance) - if err != nil { - return nil, err - } - } - - // Return volume info for the root volume only (EFI and cloud-init are internal) + // Return volume info for the root volume only (EFI is internal) volumeInfos := []VolumeInfo{ { VolumeId: imageId, @@ -1710,316 +1448,6 @@ func instanceArchitecture(it *ec2.InstanceTypeInfo) string { return *it.ProcessorInfo.SupportedArchitectures[0] } -// prepareCloudInitVolume creates the cloud-init ISO with SSH keys and user data. -// Uses rootVolumeId (not AMI ID) so each instance gets its own volume. -func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInput, rootVolumeId string, volumeConfig viperblock.VolumeConfig, instance *vm.VM) error { - slog.Info("Creating cloud-init volume") - - cloudInitVolumeName := fmt.Sprintf("%s-cloudinit", rootVolumeId) - cloudInitSize := 1 * 1024 * 1024 // 1MB - - // Update VolumeID to match the cloud-init volume name - cloudInitVolumeConfig := volumeConfig - cloudInitVolumeConfig.VolumeMetadata.VolumeID = cloudInitVolumeName - - cloudInitVb, err := s.newViperblock(cloudInitVolumeName, cloudInitSize, cloudInitVolumeConfig) - if err != nil { - slog.Error("Could not create cloudinit viperblock", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Initialize the backend - slog.Debug("Initializing cloud-init Viperblock store backend") - err = cloudInitVb.Backend.Init() - if err != nil { - slog.Error("Could not init backend", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Always create a fresh cloud-init ISO — never reuse a cached volume. - // Each instance needs its own SSH key, hostname, and user data. - - // Open the chunk WAL (sharded or legacy) - if cloudInitVb.UseShardedWAL { - err = cloudInitVb.OpenShardedWAL() - } else { - err = cloudInitVb.OpenWAL(&cloudInitVb.WAL, fmt.Sprintf("%s/%s", cloudInitVb.WAL.BaseDir, types.GetFilePath(types.FileTypeWALChunk, cloudInitVb.WAL.WallNum.Load(), cloudInitVb.GetVolume()))) - } - if err != nil { - slog.Error("Failed to load WAL", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - err = cloudInitVb.OpenWAL(&cloudInitVb.BlockToObjectWAL, fmt.Sprintf("%s/%s", cloudInitVb.WAL.BaseDir, types.GetFilePath(types.FileTypeWALBlock, cloudInitVb.BlockToObjectWAL.WallNum.Load(), cloudInitVb.GetVolume()))) - if err != nil { - slog.Error("Failed to load block WAL", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Create the cloud-init ISO - err = s.createCloudInitISO(input, instance, cloudInitVb) - if err != nil { - return err - } - - err = cloudInitVb.Close() - if err != nil { - slog.Error("Failed to close cloud-init Viperblock store", "err", err) - } - - err = cloudInitVb.RemoveLocalFiles() - if err != nil { - slog.Error("Failed to remove local files", "err", err) - } - - instance.EBSRequests.Mu.Lock() - instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, spxtypes.EBSRequest{ - Name: cloudInitVolumeName, - Boot: false, - CloudInit: true, - }) - instance.EBSRequests.Mu.Unlock() - - return nil -} - -// createCloudInitISO generates the cloud-init ISO image -func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, instance *vm.VM, cloudInitVb *viperblock.VB) error { - // Create ISO writer - writer, err := iso9660.NewWriter() - if err != nil { - slog.Error("failed to create writer", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - defer writer.Cleanup() - - // Generate instance metadata - hostname := generateHostname(instance.ID) - - // Retrieve SSH pubkey from S3 — password auth is not supported. - keyName := "" - if input.KeyName != nil { - keyName = *input.KeyName - } - - var sshKey []byte - if keyName != "" { - keyPath := fmt.Sprintf("keys/%s/%s", instance.AccountID, keyName) - result, err := s.objectStore.GetObject(&awss3.GetObjectInput{ - Bucket: aws.String(s.config.Predastore.Bucket), - Key: aws.String(keyPath), - }) - if err != nil { - if objectstore.IsNoSuchKeyError(err) { - slog.Error("key pair not found", "keyName", keyName, "err", err) - return errors.New(awserrors.ErrorInvalidKeyPairNotFound) - } - slog.Error("failed to read SSH key", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - sshKey, err = io.ReadAll(result.Body) - if err != nil { - slog.Error("failed to read SSH key body", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - } - - // Read CA cert from /config/ca.pem and inject into guest cloud-init. - var caCertPEM string - dataRoot := filepath.Dir(strings.TrimSuffix(s.config.BaseDir, "/")) - caCertPath := filepath.Join(dataRoot, "config", "ca.pem") - if caBytes, err := os.ReadFile(caCertPath); err == nil { - // Indent each line by 6 spaces for YAML block scalar in ca_certs.trusted. - var indented strings.Builder - for line := range strings.SplitSeq(string(caBytes), "\n") { - if line != "" { - indented.WriteString(" ") - indented.WriteString(line) - indented.WriteByte('\n') - } - } - caCertPEM = indented.String() - } else if os.IsNotExist(err) { - slog.Warn("CA cert not found, guest VMs will not trust Spinifex services", "path", caCertPath) - } else { - slog.Error("failed to read CA cert for guest cloud-init injection", "path", caCertPath, "error", err) - } - - // Re-read AMI metadata for DistroFamily; not persisted on VM since the ISO is sealed once. - var distroFamily string - if s.amiLoader != nil && input.ImageId != nil && *input.ImageId != "" { - amiMeta, err := s.amiLoader.GetAMIConfig(*input.ImageId) - if err != nil { - slog.Warn("createCloudInitISO: AMI metadata fetch failed; cloud-init will render with debian-family defaults (RHEL guests will boot without networking and with wrong sudo group)", "imageId", *input.ImageId, "err", err) - } else { - distroFamily = amiMeta.DistroFamily - } - } - - extraMACs := make([]string, 0, len(instance.ExtraENIs)) - for _, extra := range instance.ExtraENIs { - extraMACs = append(extraMACs, extra.ENIMac) - } - - sudoGroup := "sudo" - var rhelWriteFiles, rhelRunCmd string - switch distroFamily { - case "rhel": - sudoGroup = "wheel" - rhelWriteFiles, rhelRunCmd = buildRHELCloudInit(instance.ENIMac, instance.DevMAC, instance.MgmtMAC, instance.MgmtIP, extraMACs) - } - - userData := CloudInitData{ - Username: "ec2-user", - SSHKey: string(sshKey), - Hostname: hostname, - CACertPEM: caCertPEM, - DistroFamily: distroFamily, - SudoGroup: sudoGroup, - RHELWriteFiles: rhelWriteFiles, - RHELRunCmd: rhelRunCmd, - AlpineUnlockPasswd: distroFamily == "alpine", - } - - // Decode and classify user-data from RunInstances (base64-encoded). - if input.UserData != nil && *input.UserData != "" { - decoded, decErr := base64.StdEncoding.DecodeString(*input.UserData) - if decErr != nil { - slog.Warn("Failed to decode user-data, ignoring", "err", decErr) - } else { - raw := string(decoded) - if after, ok := strings.CutPrefix(raw, "#cloud-config"); ok { - // Strip the #cloud-config header — the template already has it - stripped := after - userData.UserDataCloudConfig = strings.TrimSpace(stripped) - } else { - // Script — indent each line by 4 spaces for YAML write_files block - var indented strings.Builder - for line := range strings.SplitSeq(raw, "\n") { - indented.WriteString(" " + line + "\n") - } - userData.UserDataScript = indented.String() - } - } - } - - var buf bytes.Buffer - t := template.Must(template.New("cloud-init").Parse(cloudInitUserDataTemplate)) - - if err := t.Execute(&buf, userData); err != nil { - slog.Error("failed to render template", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Add user-data - err = writer.AddFile(&buf, "user-data") - if err != nil { - slog.Error("failed to add file", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Add meta-data - metaData := CloudInitMetaData{ - InstanceID: instance.ID, - Hostname: hostname, - } - - t = template.Must(template.New("meta-data").Parse(cloudInitMetaTemplate)) - buf.Reset() - - if err := t.Execute(&buf, metaData); err != nil { - slog.Error("failed to render template", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - err = writer.AddFile(&buf, "meta-data") - if err != nil { - slog.Error("failed to add file", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Add network-config: per-interface for VPC+dev, wildcard DHCP otherwise. - // RHEL gets "config: disabled" to prevent a competing NM keyfile. - networkConfig := selectNetworkConfigForFamily(distroFamily, instance.ENIMac, instance.DevMAC, instance.MgmtMAC, instance.MgmtIP, extraMACs) - err = writer.AddFile(strings.NewReader(networkConfig), "network-config") - if err != nil { - slog.Error("failed to add network-config file", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Store temp file - tempFile, err := os.CreateTemp("", "cloud-init-*.iso") - if err != nil { - slog.Error("Could not create cloud-init temp file", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - slog.Info("Created temp ISO file", "file", tempFile.Name()) - - outputFile, err := os.OpenFile(tempFile.Name(), os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600) - if err != nil { - slog.Error("failed to create file", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Requires cidata volume label for cloud-init to recognize - err = writer.WriteTo(outputFile, "cidata") - if err != nil { - slog.Error("failed to write ISO image", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - err = writer.Cleanup() - if err != nil { - slog.Error("failed to cleanup writer", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - err = outputFile.Close() - if err != nil { - slog.Error("failed to close output file", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - isoData, err := os.ReadFile(tempFile.Name()) - if err != nil { - slog.Error("failed to read ISO image:", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - err = cloudInitVb.WriteAt(0, isoData) - if err != nil { - slog.Error("failed to write ISO image to viperblock volume", "err", err) - return errors.New(awserrors.ErrorServerInternal) - } - - // Flush - if err := cloudInitVb.Flush(); err != nil { - slog.Error("Failed to flush cloud-init volume", "err", err) - } - if err := cloudInitVb.WriteWALToChunk(true); err != nil { - slog.Error("Failed to write WAL to chunk", "err", err) - } - - // Remove the temp ISO file - err = os.Remove(tempFile.Name()) - if err != nil { - slog.Error("Failed to remove temp file", "err", err) - } - - return nil -} - -// generateHostname creates a hostname based on instance ID -func generateHostname(instanceID string) string { - if len(instanceID) > 2 { - uniquePart := instanceID[2:10] // Take first 8 chars after "i-" - return fmt.Sprintf("spinifex-vm-%s", uniquePart) - } - return "spinifex-vm-unknown" -} - // DescribeInstancesValidFilters lists the filter names accepted by DescribeInstances // (and the stopped/terminated variants, which share the same filter shape). var DescribeInstancesValidFilters = map[string]bool{ @@ -2642,8 +2070,8 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(instance *vm.VM, instanceID instance.EBSRequests.Mu.Lock() defer instance.EBSRequests.Mu.Unlock() for _, ebsRequest := range instance.EBSRequests.Requests { - // Internal volumes (EFI, cloud-init) are always cleaned up via ebs.delete. - if ebsRequest.EFI || ebsRequest.CloudInit { + // Internal volumes (EFI) are always cleaned up via ebs.delete. + if ebsRequest.EFI { ebsDeleteData, err := json.Marshal(spxtypes.EBSDeleteRequest{Volume: ebsRequest.Name}) if err != nil { slog.Error("TerminateStoppedInstance: failed to marshal ebs.delete request", "name", ebsRequest.Name, "err", err) diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index 09afc51c3..e75bfceeb 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -1,13 +1,11 @@ package handlers_ec2_instance import ( - "bytes" "encoding/json" "errors" "fmt" "strings" "testing" - "text/template" "time" "github.com/aws/aws-sdk-go/aws" @@ -37,42 +35,6 @@ func mgrWith(vms map[string]*vm.VM) *vm.Manager { return m } -func TestGenerateHostname(t *testing.T) { - tests := []struct { - name string - instanceID string - want string - }{ - { - name: "Normal instance ID", - instanceID: "i-0123456789abcdef0", - want: "spinifex-vm-01234567", - }, - { - name: "Too short (2 chars)", - instanceID: "ab", - want: "spinifex-vm-unknown", - }, - { - name: "Empty string", - instanceID: "", - want: "spinifex-vm-unknown", - }, - { - name: "Exactly 10 chars", - instanceID: "i-abcdef01", - want: "spinifex-vm-abcdef01", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := generateHostname(tt.instanceID) - assert.Equal(t, tt.want, got) - }) - } -} - func TestRunInstance_Success(t *testing.T) { instanceTypes := map[string]*ec2.InstanceTypeInfo{ "t3.micro": {InstanceType: aws.String("t3.micro")}, @@ -309,109 +271,6 @@ func TestRunInstance_UniqueIDs(t *testing.T) { assert.NotEqual(t, instance1.ID, instance2.ID, "Each instance should have a unique ID") } -func TestCloudInitTemplateRendering(t *testing.T) { - tests := []struct { - name string - data CloudInitData - contains []string - notContains []string - }{ - { - name: "Basic SSH key and hostname", - data: CloudInitData{ - Username: "ec2-user", - SSHKey: "ssh-rsa AAAAB3... user@host", - Hostname: "spinifex-vm-01234567", - }, - contains: []string{ - "ec2-user", - "ssh-rsa AAAAB3... user@host", - "spinifex-vm-01234567", - "#cloud-config", - }, - }, - { - name: "With cloud-config userdata", - data: CloudInitData{ - Username: "ec2-user", - SSHKey: "ssh-ed25519 AAAA...", - Hostname: "spinifex-vm-abcdef01", - UserDataCloudConfig: "packages:\n - nginx", - }, - contains: []string{ - "packages:", - "nginx", - }, - }, - { - name: "With script userdata", - data: CloudInitData{ - Username: "ec2-user", - SSHKey: "ssh-rsa AAAA...", - Hostname: "spinifex-vm-test", - UserDataScript: " #!/bin/bash\n echo hello", - }, - contains: []string{ - "write_files:", - "/tmp/cloud-init-startup.sh", - "runcmd:", - "echo hello", - }, - }, - { - name: "With CA certificate PEM", - data: CloudInitData{ - Username: "ec2-user", - SSHKey: "ssh-rsa AAAA...", - Hostname: "spinifex-vm-ca-test", - CACertPEM: " -----BEGIN CERTIFICATE-----\n" + - " MIIFazCCA1OgAwIBAgIUAbcdefg1234567890ABCDEFG=\n" + - " -----END CERTIFICATE-----\n", - }, - contains: []string{ - "ca_certs:", - "trusted:", - "-----BEGIN CERTIFICATE-----", - "-----END CERTIFICATE-----", - "MIIFazCCA1OgAwIBAgIUAbcdefg1234567890ABCDEFG=", - }, - }, - { - name: "Without CA certificate PEM", - data: CloudInitData{ - Username: "ec2-user", - SSHKey: "ssh-rsa AAAA...", - Hostname: "spinifex-vm-no-ca", - }, - contains: []string{ - "#cloud-config", - "spinifex-vm-no-ca", - }, - notContains: []string{ - "ca_certs:", - "trusted:", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tmpl := template.Must(template.New("cloud-init").Parse(cloudInitUserDataTemplate)) - var buf bytes.Buffer - err := tmpl.Execute(&buf, tt.data) - require.NoError(t, err) - - rendered := buf.String() - for _, s := range tt.contains { - assert.Contains(t, rendered, s) - } - for _, s := range tt.notContains { - assert.NotContains(t, rendered, s) - } - }) - } -} - func TestFloorVolumeSizeToAMI(t *testing.T) { loader := &fakeAMILoader{byID: map[string]viperblock.AMIMetadata{ "ami-rocky": {VolumeSizeGiB: 10}, @@ -445,213 +304,6 @@ func TestFloorVolumeSizeToAMI(t *testing.T) { }) } -func TestBuildRHELCloudInit(t *testing.T) { - t.Run("empty eniMAC returns empty (wildcard path)", func(t *testing.T) { - wf, rc := buildRHELCloudInit("", "", "", "", nil) - assert.Empty(t, wf) - assert.Empty(t, rc) - }) - - t.Run("vpc0 only", func(t *testing.T) { - wf, rc := buildRHELCloudInit("02:00:00:00:00:01", "", "", "", nil) - // File mode + ownership are load-bearing — NM ignores anything else. - assert.Contains(t, wf, "/etc/NetworkManager/system-connections/vpc0.nmconnection") - assert.Contains(t, wf, "owner: root:root") - assert.Contains(t, wf, "permissions: '0600'") - assert.Contains(t, wf, "mac-address=02:00:00:00:00:01") - assert.Contains(t, wf, "dhcp-client-id=mac") - assert.Contains(t, wf, "method=auto") - assert.NotContains(t, wf, "never-default") - // IMDS is served at the host tap, so the NM keyfile carries no on-link route. - assert.NotContains(t, wf, "route1=169.254.169.254/32", "vpc0 must not carry an in-guest IMDS route") - assert.Contains(t, rc, " - [ restorecon, -R, /etc/NetworkManager/system-connections/ ]") - assert.Contains(t, rc, " - [ nmcli, connection, reload ]") - assert.Contains(t, rc, " - [ nmcli, connection, up, vpc0 ]") - }) - - t.Run("vpc0 + extra ENIs + dev + mgmt", func(t *testing.T) { - wf, rc := buildRHELCloudInit( - "02:00:00:00:00:01", - "02:00:00:00:00:99", - "02:00:00:00:00:aa", "10.250.0.5", - []string{"02:00:00:00:00:02", "02:00:00:00:00:03"}, - ) - for _, name := range []string{"vpc0", "vpc1", "vpc2", "dev0", "mgmt0"} { - assert.Contains(t, wf, "/etc/NetworkManager/system-connections/"+name+".nmconnection", name) - assert.Contains(t, rc, " - [ nmcli, connection, up, "+name+" ]", name) - } - // dev NIC must not install default route or DNS. - assert.Contains(t, wf, "never-default=true") - assert.Contains(t, wf, "ignore-auto-dns=true") - // mgmt NIC is static with the supplied IP. - assert.Contains(t, wf, "method=manual") - assert.Contains(t, wf, "addresses=10.250.0.5/24") - }) - - t.Run("empty extra MAC skipped", func(t *testing.T) { - wf, _ := buildRHELCloudInit( - "02:00:00:00:00:01", "", "", "", - []string{"", "02:00:00:00:00:02"}, - ) - // Empty slot is skipped, but index advances — second valid MAC is vpc2. - assert.NotContains(t, wf, "vpc1.nmconnection") - assert.Contains(t, wf, "vpc2.nmconnection") - }) - - t.Run("no NIC carries an in-guest IMDS route", func(t *testing.T) { - wf, _ := buildRHELCloudInit( - "02:00:00:00:00:01", - "02:00:00:00:00:99", - "", "", - []string{"02:00:00:00:00:02"}, - ) - // Served at the tap — no NM keyfile (primary, extra, or dev) emits a route. - assert.NotContains(t, wf, "route1=169.254.169.254/32") - }) -} - -func TestCloudInitTemplate_FamilyBranching(t *testing.T) { - rhelWF, rhelRC := buildRHELCloudInit("02:00:00:11:22:33", "", "", "", nil) - - t.Run("debian family: sudo group, no NM keyfiles", func(t *testing.T) { - var buf bytes.Buffer - tmpl := template.Must(template.New("c").Parse(cloudInitUserDataTemplate)) - require.NoError(t, tmpl.Execute(&buf, CloudInitData{ - Username: "ec2-user", SSHKey: "ssh-rsa AAA", Hostname: "spinifex-vm-deb", - DistroFamily: "debian", SudoGroup: "sudo", - })) - out := buf.String() - assert.Contains(t, out, "- sudo") - assert.NotContains(t, out, "- wheel") - assert.NotContains(t, out, "NetworkManager/system-connections") - assert.NotContains(t, out, "nmcli") - assert.NotContains(t, out, "restorecon") - }) - - t.Run("rhel family: wheel group, NM keyfile write_files + runcmd", func(t *testing.T) { - var buf bytes.Buffer - tmpl := template.Must(template.New("c").Parse(cloudInitUserDataTemplate)) - require.NoError(t, tmpl.Execute(&buf, CloudInitData{ - Username: "ec2-user", SSHKey: "ssh-rsa AAA", Hostname: "spinifex-vm-rhel", - DistroFamily: "rhel", SudoGroup: "wheel", - RHELWriteFiles: rhelWF, RHELRunCmd: rhelRC, - })) - out := buf.String() - assert.Contains(t, out, "- wheel") - assert.NotContains(t, out, "- sudo\n") - assert.Contains(t, out, "write_files:") - assert.Contains(t, out, "/etc/NetworkManager/system-connections/vpc0.nmconnection") - assert.Contains(t, out, "permissions: '0600'") - assert.Contains(t, out, "dhcp-client-id=mac") - assert.Contains(t, out, "runcmd:") - assert.Contains(t, out, "restorecon") - assert.Contains(t, out, "nmcli, connection, reload") - assert.Contains(t, out, "nmcli, connection, up, vpc0") - }) - - t.Run("alpine family: user rendered unlocked with disabled password", func(t *testing.T) { - var buf bytes.Buffer - tmpl := template.Must(template.New("c").Parse(cloudInitUserDataTemplate)) - require.NoError(t, tmpl.Execute(&buf, CloudInitData{ - Username: "ec2-user", SSHKey: "ssh-rsa AAA", Hostname: "spinifex-vm-alpine", - DistroFamily: "alpine", SudoGroup: "sudo", AlpineUnlockPasswd: true, - })) - out := buf.String() - // Alpine's no-PAM sshd denies a locked ("!") account; "*" keeps password - // auth disabled while letting pubkey auth through. - assert.Contains(t, out, "lock_passwd: false") - assert.Contains(t, out, `hashed_passwd: "*"`) - }) - - t.Run("non-alpine family: no password unlock lines", func(t *testing.T) { - var buf bytes.Buffer - tmpl := template.Must(template.New("c").Parse(cloudInitUserDataTemplate)) - require.NoError(t, tmpl.Execute(&buf, CloudInitData{ - Username: "ec2-user", SSHKey: "ssh-rsa AAA", Hostname: "spinifex-vm-deb", - DistroFamily: "debian", SudoGroup: "sudo", - })) - out := buf.String() - assert.NotContains(t, out, "lock_passwd") - assert.NotContains(t, out, "hashed_passwd") - }) - - t.Run("rhel + user script: both blocks merged under single write_files/runcmd", func(t *testing.T) { - var buf bytes.Buffer - tmpl := template.Must(template.New("c").Parse(cloudInitUserDataTemplate)) - require.NoError(t, tmpl.Execute(&buf, CloudInitData{ - Username: "ec2-user", SSHKey: "ssh-rsa AAA", Hostname: "spinifex-vm-rhel", - DistroFamily: "rhel", SudoGroup: "wheel", - RHELWriteFiles: rhelWF, RHELRunCmd: rhelRC, - UserDataScript: " #!/bin/bash\n echo hi\n", - })) - out := buf.String() - // YAML disallows duplicate top-level keys; assert each appears exactly once. - assert.Equal(t, 1, strings.Count(out, "\nwrite_files:")) - assert.Equal(t, 1, strings.Count(out, "\nruncmd:")) - assert.Contains(t, out, "/etc/NetworkManager/system-connections/vpc0.nmconnection") - assert.Contains(t, out, "/tmp/cloud-init-startup.sh") - assert.Contains(t, out, "nmcli, connection, up, vpc0") - assert.Contains(t, out, `"/bin/bash", "/tmp/cloud-init-startup.sh"`) - }) -} - -func TestSelectNetworkConfigForFamily(t *testing.T) { - const eniMAC = "02:00:00:00:00:01" - - t.Run("rhel returns disabled stub", func(t *testing.T) { - out := selectNetworkConfigForFamily("rhel", eniMAC, "", "", "", nil) - assert.Equal(t, cloudInitNetworkConfigDisabled, out) - assert.Contains(t, out, "config: disabled") - // Must not emit any netplan v2 keys — those would render alongside - // our NM keyfile and produce a competing cloud-init-enpXsY connection. - assert.NotContains(t, out, "ethernets:") - assert.NotContains(t, out, "version: 2") - }) - - t.Run("debian returns per-interface netplan", func(t *testing.T) { - out := selectNetworkConfigForFamily("debian", eniMAC, "", "", "", nil) - assert.Contains(t, out, "version: 2") - assert.Contains(t, out, "ethernets:") - assert.Contains(t, out, "vpc0:") - assert.Contains(t, out, "dhcp-identifier: mac") - assert.NotContains(t, out, "config: disabled") - }) - - t.Run("empty family falls through to netplan", func(t *testing.T) { - // Legacy AMIs without DistroFamily populated must keep today's behaviour. - out := selectNetworkConfigForFamily("", eniMAC, "", "", "", nil) - assert.Contains(t, out, "version: 2") - assert.NotContains(t, out, "config: disabled") - }) - - t.Run("alpine falls through to netplan", func(t *testing.T) { - // Alpine has no RHEL-style NM keyfile path; reuses netplan. - out := selectNetworkConfigForFamily("alpine", eniMAC, "", "", "", nil) - assert.Contains(t, out, "version: 2") - assert.NotContains(t, out, "config: disabled") - }) -} - -func TestCloudInitMetaTemplateRendering(t *testing.T) { - data := CloudInitMetaData{ - InstanceID: "i-0123456789abcdef0", - Hostname: "spinifex-vm-01234567", - } - - tmpl := template.Must(template.New("meta-data").Parse(cloudInitMetaTemplate)) - var buf bytes.Buffer - err := tmpl.Execute(&buf, data) - require.NoError(t, err) - - rendered := buf.String() - assert.Contains(t, rendered, "i-0123456789abcdef0") - assert.Contains(t, rendered, "spinifex-vm-01234567") - assert.Contains(t, rendered, "instance-id:") - assert.Contains(t, rendered, "local-hostname:") -} - -// TestCloudInitVolumeNamePerInstance verifies that each launch produces a unique -// root volume ID so cloud-init ISOs are not shared across instances. func TestRunInstance_NoImageId(t *testing.T) { instanceTypes := map[string]*ec2.InstanceTypeInfo{ "t3.micro": {InstanceType: aws.String("t3.micro")}, @@ -762,80 +414,6 @@ func TestParseVolumeParams_PartialEbs(t *testing.T) { assert.True(t, p.deleteOnTermination, "deleteOnTermination should stay at default") } -func TestCloudInitNetworkConfigWildcard(t *testing.T) { - // No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING) - cfg := generateNetworkConfig("", "", "", "", nil) - assert.Contains(t, cfg, "version: 2") - assert.Contains(t, cfg, "dhcp4: true") - assert.Contains(t, cfg, "dhcp-identifier: mac") - assert.Contains(t, cfg, `name: "e*"`, "wildcard should match both eth* and en* interfaces") - assert.NotContains(t, cfg, "use-routes") -} - -func TestCloudInitNetworkConfigDualNIC(t *testing.T) { - eniMAC := "02:00:00:61:ef:c2" - devMAC := "02:de:00:60:83:0d" - - cfg := generateNetworkConfig(eniMAC, devMAC, "", "", nil) - - // Both MACs present in config - assert.Contains(t, cfg, eniMAC) - assert.Contains(t, cfg, devMAC) - - // VPC NIC gets normal DHCP (with default route) - assert.Contains(t, cfg, "vpc0:") - assert.Contains(t, cfg, "dev0:") - - // Dev NIC has route/DNS suppressed - assert.Contains(t, cfg, "use-routes: false") - assert.Contains(t, cfg, "use-dns: false") - - // No wildcard match — per-interface only - assert.NotContains(t, cfg, `name: "e*"`) -} - -func TestCloudInitNetworkConfigPartialMAC(t *testing.T) { - // Only ENI MAC (VPC without dev) → per-interface config with VPC NIC only - cfg := generateNetworkConfig("02:00:00:61:ef:c2", "", "", "", nil) - assert.Contains(t, cfg, "vpc0:") - assert.NotContains(t, cfg, "dev0:") - - // Only dev MAC (shouldn't happen, but defensive) → wildcard - cfg = generateNetworkConfig("", "02:de:00:60:83:0d", "", "", nil) - assert.Contains(t, cfg, `name: "e*"`) - assert.NotContains(t, cfg, "use-routes") -} - -func TestCloudInitNetworkConfigMultiVPCNICs(t *testing.T) { - // Multi-subnet ALB VM: primary ENI + two extras, each on a different AZ. - extras := []string{ - "02:00:00:bb:bb:bb", - "02:00:00:cc:cc:cc", - } - cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", extras) - - assert.Contains(t, cfg, "vpc0:") - assert.Contains(t, cfg, "vpc1:") - assert.Contains(t, cfg, "vpc2:") - assert.Contains(t, cfg, `macaddress: "02:00:00:aa:aa:aa"`) - assert.Contains(t, cfg, `macaddress: "02:00:00:bb:bb:bb"`) - assert.Contains(t, cfg, `macaddress: "02:00:00:cc:cc:cc"`) - // Each extra NIC gets its own DHCP block with dhcp-identifier: mac. - assert.Equal(t, 3, strings.Count(cfg, "dhcp4: true")) - assert.Equal(t, 3, strings.Count(cfg, "dhcp-identifier: mac")) - // No dev0 / mgmt0 unless explicitly configured. - assert.NotContains(t, cfg, "dev0:") - assert.NotContains(t, cfg, "mgmt0:") -} - -func TestCloudInitNetworkConfigEmptyExtraMACSkipped(t *testing.T) { - // Empty strings inside the extras slice are ignored rather than producing - // a malformed ethernets block. - cfg := generateNetworkConfig("02:00:00:aa:aa:aa", "", "", "", []string{""}) - assert.Contains(t, cfg, "vpc0:") - assert.NotContains(t, cfg, "vpc1:") -} - func TestRunInstance_WithTags(t *testing.T) { instanceTypes := map[string]*ec2.InstanceTypeInfo{ "t3.micro": {InstanceType: aws.String("t3.micro")}, @@ -934,30 +512,6 @@ func TestParseVolumeParams_Io1WithIops(t *testing.T) { assert.Equal(t, 5000, p.iops) } -func TestCloudInitVolumeNamePerInstance(t *testing.T) { - amiID := "ami-0abcdef1234567890" - - seen := make(map[string]bool) - for range 100 { - // Simulate GenerateVolumes logic for AMI-based launches (line 194-195) - var rootVolumeId string - if strings.HasPrefix(amiID, "ami-") { - rootVolumeId = utils.GenerateResourceID("vol") - } - - cloudInitName := fmt.Sprintf("%s-cloudinit", rootVolumeId) - - assert.True(t, strings.HasPrefix(cloudInitName, "vol-"), - "cloud-init volume should be keyed by root volume ID, not AMI ID") - assert.True(t, strings.HasSuffix(cloudInitName, "-cloudinit")) - assert.False(t, strings.Contains(cloudInitName, "ami-"), - "cloud-init volume name must not contain the AMI ID") - assert.False(t, seen[cloudInitName], - "each instance must get a unique cloud-init volume name") - seen[cloudInitName] = true - } -} - // --- Describe* coverage ----------------------------------------------------- type fakeResourceCapacityProvider struct { @@ -2123,7 +1677,6 @@ func TestTerminateStoppedInstance_InternalVolumesViaNATS(t *testing.T) { v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"} v.EBSRequests.Requests = []spxtypes.EBSRequest{ {Name: "vol-efi-001", EFI: true}, - {Name: "vol-ci-001", CloudInit: true}, } store := &fakeStoppedStore{loadByID: map[string]*vm.VM{id: v}} @@ -2142,7 +1695,7 @@ func TestTerminateStoppedInstance_InternalVolumesViaNATS(t *testing.T) { _, err = svc.TerminateStoppedInstance(&TerminateStoppedInstanceInput{InstanceID: id}, "acc") require.NoError(t, err) - assert.ElementsMatch(t, []string{"vol-efi-001", "vol-ci-001"}, ebsDeleted) + assert.ElementsMatch(t, []string{"vol-efi-001"}, ebsDeleted) } func TestTerminateStoppedInstance_PublicIPReleased(t *testing.T) { diff --git a/spinifex/handlers/ec2/volume/service_impl.go b/spinifex/handlers/ec2/volume/service_impl.go index 76cf33858..3c11fdfdc 100644 --- a/spinifex/handlers/ec2/volume/service_impl.go +++ b/spinifex/handlers/ec2/volume/service_impl.go @@ -842,7 +842,9 @@ func (s *VolumeServiceImpl) listAllVolumeIDs() ([]string, error) { volumeID := strings.TrimSuffix(prefixStr, "/") - // Skip internal sub-volumes (EFI and cloud-init partitions) + // Skip internal sub-volumes: the EFI partition and the legacy cloud-init + // seed (no longer created post boot-from-IMDS, still filtered so any + // pre-cutover volume never surfaces as a user volume). if strings.HasSuffix(volumeID, "-efi") || strings.HasSuffix(volumeID, "-cloudinit") { continue } @@ -1367,7 +1369,6 @@ func (s *VolumeServiceImpl) DeleteVolume(input *ec2.DeleteVolumeInput, accountID // available for retry if an auxiliary deletion fails. prefixes := []string{ volumeID + "-efi/", - volumeID + "-cloudinit/", volumeID + "/", } diff --git a/spinifex/handlers/imds/metadata.go b/spinifex/handlers/imds/metadata.go index cc12449bd..c3bc596a8 100644 --- a/spinifex/handlers/imds/metadata.go +++ b/spinifex/handlers/imds/metadata.go @@ -100,6 +100,10 @@ func (s *IMDSServiceImpl) handleToken(w http.ResponseWriter, r *http.Request) { return } + // First-contact log: a guest reaching this proves its packets traverse the + // per-tap datapath; its absence (with the responder bound) points at the datapath. + slog.Info("IMDS: issued IMDSv2 token", "instance_id", eni.instanceID, "private_ip", eni.privateIP, "public_ip", eni.publicIP) + w.Header().Set("Content-Type", "text/plain") w.Header().Set(hdrTokenTTL, strconv.Itoa(int(ttl.Seconds()))) _, _ = w.Write([]byte(token)) @@ -140,6 +144,11 @@ func (s *IMDSServiceImpl) resolveCaller(r *http.Request) *eniFacts { func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni *eniFacts) { path := r.URL.Path + // Boot-crawl access log: traces every metadata GET per ENI so a guest's + // cloud-init crawl is observable end-to-end (e.g. private vs public subnet). + slog.Info("IMDS: serving metadata request", "path", path, + "instance_id", eni.instanceID, "private_ip", eni.privateIP, "public_ip", eni.publicIP) + if strings.HasPrefix(path, prefixSecurityCreds) && len(path) > len(prefixSecurityCreds) { s.serveRoleCredentials(w, eni, strings.TrimPrefix(path, prefixSecurityCreds)) return @@ -167,7 +176,7 @@ func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni * case pathIdentityDocument: s.serveInstanceIdentityDocument(w, eni) case pathMetaDataRoot, prefixMetaData: - writeText(w, "ami-id\nami-launch-index\nhostname\niam/\ninstance-id\ninstance-life-cycle\ninstance-type\nlocal-hostname\nlocal-ipv4\nmac\nnetwork/\nplacement/\npublic-hostname\npublic-ipv4\npublic-keys/\nreservation-id\nsecurity-groups\nservices/") + s.serveMetaDataRoot(w, eni) case prefixMetaData + "instance-id": writeText(w, eni.instanceID) case prefixMetaData + "instance-life-cycle": @@ -175,6 +184,10 @@ func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni * case prefixMetaData + "local-ipv4": writeText(w, eni.privateIP) case prefixMetaData + "public-ipv4": + if eni.publicIP == "" { + w.WriteHeader(http.StatusNotFound) // no public IP → 404, as on real EC2 + return + } writeText(w, eni.publicIP) case prefixMetaData + "public-hostname": if eni.publicIP == "" { @@ -217,6 +230,13 @@ func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni * case prefixMetaData + "services/partition": writeText(w, "aws") case prefixMetaData + "iam", prefixMetaData + "iam/": + // A backend error counts as "no profile" so the iam/ subtree stays + // self-consistent — 404 here rather than advertised with 404ing leaves, + // which fails cloud-init's metadata crawl. + if profile, err := s.profileFor(eni); err != nil || profile == nil { + w.WriteHeader(http.StatusNotFound) // no profile → no iam/ subtree, as on real EC2 + return + } writeText(w, "info\nsecurity-credentials/") case prefixMetaData + "iam/info": s.serveIAMInfo(w, eni) @@ -231,6 +251,38 @@ func (s *IMDSServiceImpl) dispatch(w http.ResponseWriter, r *http.Request, eni * } } +// serveMetaDataRoot writes the meta-data/ index, listing only children whose leaf +// serves content for this instance so cloud-init's recursive crawl never 404s +// mid-listing and falls back to DataSourceNone: public-hostname/public-ipv4 only +// with a public IP, public-keys/ only with a key pair, iam/ only with an instance +// profile — each omission matching real EC2, like the macs/ subtree in macKeys. +func (s *IMDSServiceImpl) serveMetaDataRoot(w http.ResponseWriter, eni *eniFacts) { + keys := []string{ + "ami-id", "ami-launch-index", "hostname", "instance-id", + "instance-life-cycle", "instance-type", "local-hostname", "local-ipv4", + "mac", "network/", "placement/", "reservation-id", "security-groups", + "services/", + } + if eni.publicIP != "" { + keys = append(keys, "public-hostname", "public-ipv4") + } + // Resolve the instance once: public-keys/ is listed only with a key pair, iam/ + // only with a resolvable instance profile. A backend error counts as absent so + // the listing never advertises a child whose leaf would 404 and break the crawl. + if inst, err := s.resolver.resolveInstance(eni); err == nil && inst != nil { + if inst.keyName != "" { + keys = append(keys, "public-keys/") + } + if inst.iamInstanceProfileArn != "" { + if p, err := s.iam.ResolveInstanceProfile(eni.accountID, inst.iamInstanceProfileArn); err == nil && p != nil { + keys = append(keys, "iam/") + } + } + } + sort.Strings(keys) + writeText(w, strings.Join(keys, "\n")) +} + // serveInstanceField resolves the instance record and writes one of its // string fields, 404ing when the instance is no longer visible. func (s *IMDSServiceImpl) serveInstanceField(w http.ResponseWriter, eni *eniFacts, field func(*instanceFacts) string) { @@ -395,6 +447,7 @@ func (s *IMDSServiceImpl) servePublicKeys(w http.ResponseWriter, eni *eniFacts, w.WriteHeader(http.StatusInternalServerError) return } + slog.Info("IMDS: served SSH public key", "key_name", inst.keyName, "instance_id", eni.instanceID, "private_ip", eni.privateIP) writeText(w, material+"\n") default: w.WriteHeader(http.StatusNotFound) diff --git a/spinifex/handlers/imds/metadata_test.go b/spinifex/handlers/imds/metadata_test.go index f6d9c4c5b..3ce8389b2 100644 --- a/spinifex/handlers/imds/metadata_test.go +++ b/spinifex/handlers/imds/metadata_test.go @@ -425,20 +425,58 @@ func TestHTTP_MetadataPaths(t *testing.T) { } } -// An instance with no public IP has no public hostname: 404, not an empty 200. -func TestHTTP_PublicHostnameAbsent404(t *testing.T) { +// An instance with no public IP has no public hostname or public IPv4: both leaves +// 404, not an empty 200, matching real EC2 and the meta-data/ listing that omits them. +func TestHTTP_PublicFieldsAbsent404(t *testing.T) { eni := testENI() eni.publicIP = "" svc, _ := newTestService(&fakeResolver{eni: eni}, &fakeIAM{}, &fakeAssumer{}) h := withTapENI(svc.httpHandler(), eni) token := issueToken(t, h) - rec := get(t, h, prefixMetaData+"public-hostname", token) - assert.Equal(t, http.StatusNotFound, rec.Code) + for _, leaf := range []string{"public-hostname", "public-ipv4"} { + rec := get(t, h, prefixMetaData+leaf, token) + assert.Equal(t, http.StatusNotFound, rec.Code, "leaf=%s", leaf) + } } -// The meta-data root lists every served child, alphabetically, to match AWS. +// The meta-data root lists every served child, alphabetically, to match AWS. With +// no instance profile attached, iam/ is omitted entirely — real EC2 omits it so +// cloud-init never descends into a subtree whose leaves would 404 and fail the crawl. func TestHTTP_DirectoryListing(t *testing.T) { - svc, _ := newTestService(&fakeResolver{eni: testENI()}, &fakeIAM{}, &fakeAssumer{}) + res := &fakeResolver{eni: testENI(), inst: &instanceFacts{keyName: "e2e-key"}} + svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{}) + h := withTapENI(svc.httpHandler(), testENI()) + token := issueToken(t, h) + rec := get(t, h, pathMetaDataRoot, token) + assert.Equal(t, http.StatusOK, rec.Code) + want := strings.Join([]string{ + "ami-id", + "ami-launch-index", + "hostname", + "instance-id", + "instance-life-cycle", + "instance-type", + "local-hostname", + "local-ipv4", + "mac", + "network/", + "placement/", + "public-hostname", + "public-ipv4", + "public-keys/", + "reservation-id", + "security-groups", + "services/", + }, "\n") + assert.Equal(t, want, rec.Body.String()) + assert.NotContains(t, rec.Body.String(), "iam/") +} + +// With an instance profile attached, iam/ appears in the listing (between hostname +// and instance-id) so cloud-init can fetch the role credentials. +func TestHTTP_DirectoryListingWithProfile(t *testing.T) { + res := &fakeResolver{eni: testENI(), inst: &instanceFacts{keyName: "e2e-key", iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}} + svc, _ := newTestService(res, &fakeIAM{profile: profileFixture()}, &fakeAssumer{}) h := withTapENI(svc.httpHandler(), testENI()) token := issueToken(t, h) rec := get(t, h, pathMetaDataRoot, token) @@ -466,6 +504,36 @@ func TestHTTP_DirectoryListing(t *testing.T) { assert.Equal(t, want, rec.Body.String()) } +// A private-subnet instance (no public IP) omits public-hostname and public-ipv4 from +// the listing so cloud-init never fetches the 404ing leaves and falls back to +// DataSourceNone — the regression that left private VMs keyless. public-keys/ stays: +// it tracks the key pair, not the public IP. +func TestHTTP_DirectoryListingNoPublicIP(t *testing.T) { + eni := testENI() + eni.publicIP = "" + res := &fakeResolver{eni: eni, inst: &instanceFacts{keyName: "e2e-key"}} + svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{}) + h := withTapENI(svc.httpHandler(), eni) + token := issueToken(t, h) + rec := get(t, h, pathMetaDataRoot, token) + assert.Equal(t, http.StatusOK, rec.Code) + body := rec.Body.String() + assert.NotContains(t, body, "public-hostname") + assert.NotContains(t, body, "public-ipv4") + assert.Contains(t, body, "public-keys/") +} + +// An instance launched without a key pair omits public-keys/, whose leaf would 404. +func TestHTTP_DirectoryListingNoKeyPair(t *testing.T) { + res := &fakeResolver{eni: testENI(), inst: &instanceFacts{}} + svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{}) + h := withTapENI(svc.httpHandler(), testENI()) + token := issueToken(t, h) + rec := get(t, h, pathMetaDataRoot, token) + assert.Equal(t, http.StatusOK, rec.Code) + assert.NotContains(t, rec.Body.String(), "public-keys/") +} + func TestHTTP_UserDataAbsent404(t *testing.T) { res := &fakeResolver{eni: testENI(), inst: &instanceFacts{}} svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{}) @@ -792,6 +860,32 @@ func TestHTTP_IAMInfoNoProfile404(t *testing.T) { assert.Equal(t, http.StatusNotFound, rec.Code) } +// With an instance profile attached, the iam/ directory lists its children. +func TestHTTP_IAMDir(t *testing.T) { + res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}} + svc, _ := newTestService(res, &fakeIAM{profile: profileFixture()}, &fakeAssumer{}) + h := withTapENI(svc.httpHandler(), testENI()) + token := issueToken(t, h) + for _, path := range []string{prefixMetaData + "iam", prefixMetaData + "iam/"} { + rec := get(t, h, path, token) + assert.Equal(t, http.StatusOK, rec.Code, path) + assert.Equal(t, "info\nsecurity-credentials/", rec.Body.String(), path) + } +} + +// With no instance profile, the iam/ directory itself 404s — matching real EC2, so +// cloud-init never descends and never trips on a 404ing iam/info that fails its crawl. +func TestHTTP_IAMDirNoProfile404(t *testing.T) { + res := &fakeResolver{eni: testENI(), inst: &instanceFacts{}} // no profile ARN + svc, _ := newTestService(res, &fakeIAM{}, &fakeAssumer{}) + h := withTapENI(svc.httpHandler(), testENI()) + token := issueToken(t, h) + for _, path := range []string{prefixMetaData + "iam", prefixMetaData + "iam/"} { + rec := get(t, h, path, token) + assert.Equal(t, http.StatusNotFound, rec.Code, path) + } +} + func TestHTTP_SecurityCredentialsList(t *testing.T) { res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/app-profile"}} svc, _ := newTestService(res, &fakeIAM{profile: profileFixture()}, &fakeAssumer{}) diff --git a/spinifex/handlers/imds/tap_responder.go b/spinifex/handlers/imds/tap_responder.go index 7e4cc4765..2332aec0c 100644 --- a/spinifex/handlers/imds/tap_responder.go +++ b/spinifex/handlers/imds/tap_responder.go @@ -23,11 +23,19 @@ type tapListenFunc func(ctx context.Context, endpoint string) (net.Listener, err // responder manager is unit-testable without a live ENI bucket. type resolveENIFunc func(eniID string) (*eniFacts, error) +// ifindexFunc resolves an endpoint device's current kernel ifindex. Injected so +// the recreated-endpoint check is unit-testable without real devices. +type ifindexFunc func(dev string) (int, error) + // activeTapResponder is one tap's realised serving state: a listener bound to the // tap's endpoint on br-imds and the http.Server serving it with the tap's ENI. +// endpoint + ifindex pin which device the listener is bound to (via +// SO_BINDTODEVICE), so reconcile can spot a torn-down/recreated endpoint. type activeTapResponder struct { listener net.Listener server *http.Server + endpoint string + ifindex int } // tapResponderManager runs one IMDS responder per local primary-ENI tap. Each @@ -37,6 +45,7 @@ type tapResponderManager struct { handler http.Handler resolve resolveENIFunc listen tapListenFunc + ifindex ifindexFunc mu sync.Mutex active map[string]*activeTapResponder // eniID → responder @@ -47,6 +56,7 @@ func newTapResponderManager(handler http.Handler, resolve resolveENIFunc, listen handler: handler, resolve: resolve, listen: listen, + ifindex: deviceIfindex, active: make(map[string]*activeTapResponder), } } @@ -60,11 +70,19 @@ func (m *tapResponderManager) start(ctx context.Context, eniID, endpoint string) } m.mu.Lock() - if _, ok := m.active[eniID]; ok { - m.mu.Unlock() - return nil - } + cur, ok := m.active[eniID] m.mu.Unlock() + if ok { + if !m.endpointRecreated(cur, endpoint) { + return nil + } + // A stop/start faster than the reconcile interval recreates the endpoint + // device (same ENI-derived name, fresh ifindex) without reconcile ever + // seeing the gap, so this stale listener stays bound to the deleted device + // via SO_BINDTODEVICE and serves nothing. Drop it and rebind to the live one. + slog.Info("IMDS: tap endpoint recreated, rebinding responder", "eni_id", eniID, "endpoint", endpoint) + m.stop(eniID) + } eni, err := m.resolve(eniID) if err != nil { @@ -78,6 +96,10 @@ func (m *tapResponderManager) start(ctx context.Context, eniID, endpoint string) if err != nil { return fmt.Errorf("listen on endpoint %s: %w", endpoint, err) } + ifindex, err := m.ifindex(endpoint) + if err != nil { + slog.Debug("IMDS: endpoint ifindex unavailable; recreated-endpoint detection degraded", "endpoint", endpoint, "err", err) + } server := &http.Server{ Handler: m.handler, @@ -93,7 +115,7 @@ func (m *tapResponderManager) start(ctx context.Context, eniID, endpoint string) _ = listener.Close() return nil } - m.active[eniID] = &activeTapResponder{listener: listener, server: server} + m.active[eniID] = &activeTapResponder{listener: listener, server: server, endpoint: endpoint, ifindex: ifindex} m.mu.Unlock() go func() { @@ -111,6 +133,26 @@ func (m *tapResponderManager) start(ctx context.Context, eniID, endpoint string) return nil } +// endpointRecreated reports whether eniID's live endpoint device differs from the +// one its responder is bound to — the fingerprint of a stop/start the reconcile +// interval missed, where the endpoint port was torn down and re-added under the +// same ENI-derived name with a fresh ifindex. An ifindex that is unresolvable now +// or was unresolvable at bind (0) is treated as unchanged, so a transient lookup +// miss never churns a healthy responder. +func (m *tapResponderManager) endpointRecreated(cur *activeTapResponder, endpoint string) bool { + if cur.endpoint != endpoint { + return true + } + if cur.ifindex == 0 { + return false + } + idx, err := m.ifindex(endpoint) + if err != nil { + return false + } + return idx != cur.ifindex +} + // reconcile converges active responders to the live tap set: starts one for every // (eniID → endpoint) not yet serving and stops any whose tap has gone. A start // failure is logged and retried next reconcile so one stalled tap can't block the rest. @@ -175,6 +217,17 @@ func (m *tapResponderManager) shutdown() { } } +// deviceIfindex resolves dev's current kernel ifindex. A torn-down/recreated +// endpoint reappears under the same name with a fresh ifindex, which is how a +// responder bound to the old device (via SO_BINDTODEVICE) is detected as stale. +func deviceIfindex(dev string) (int, error) { + iface, err := net.InterfaceByName(dev) + if err != nil { + return 0, err + } + return iface.Index, nil +} + // bindTapListener opens 169.254.169.254:80 on the tap's endpoint via SO_BINDTODEVICE // — the per-tap serving socket. No netns/CAP_SYS_ADMIN: the endpoint is in the root // netns on br-imds and owns .254, so the bind targets the address the guest does. diff --git a/spinifex/handlers/imds/tap_responder_test.go b/spinifex/handlers/imds/tap_responder_test.go index 3654c05ae..336de5d51 100644 --- a/spinifex/handlers/imds/tap_responder_test.go +++ b/spinifex/handlers/imds/tap_responder_test.go @@ -192,6 +192,48 @@ func TestTapResponder_UnexpectedServeExitAllowsRestart(t *testing.T) { m.shutdown() } +// TestTapResponder_RebindsOnRecreatedEndpoint guards the fix for a stop/start +// faster than the reconcile interval: the endpoint device is torn down and +// re-added under the same ENI-derived name with a fresh ifindex, but reconcile +// never sees the gap. The stale listener stays bound to the deleted device via +// SO_BINDTODEVICE and serves nothing, so start must detect the new ifindex and +// rebind rather than no-op on the still-present eniID. +func TestTapResponder_RebindsOnRecreatedEndpoint(t *testing.T) { + const ( + eniID = "eni-aaa11111" + endpoint = "ime-aaa11111" + ) + table := map[string]*eniFacts{eniID: {eniID: eniID, instanceID: "i-aaaa1111"}} + + var addrs sync.Map + m := newTapResponderManager(http.NewServeMux(), staticResolve(table), loopbackTapListen(&addrs)) + // Controllable endpoint ifindex; bumped to mimic a torn-down/recreated device. + ifindex := 10 + m.ifindex = func(string) (int, error) { return ifindex, nil } + ctx := context.Background() + + require.NoError(t, m.start(ctx, eniID, endpoint)) + addrOld := mustAddr(t, &addrs, endpoint) + + // Same endpoint, unchanged ifindex: idempotent no-op, listener not rebound. + require.NoError(t, m.start(ctx, eniID, endpoint)) + assert.Equal(t, addrOld, mustAddr(t, &addrs, endpoint), "unchanged endpoint must not rebind") + + // Endpoint recreated (new ifindex): start must drop the stale responder and + // rebind to a fresh listener on the live device. + ifindex = 20 + require.NoError(t, m.start(ctx, eniID, endpoint)) + addrNew := mustAddr(t, &addrs, endpoint) + assert.NotEqual(t, addrOld, addrNew, "recreated endpoint must rebind to a fresh listener") + + // The stale listener bound to the old device must be closed. + client := http.Client{Timeout: 500 * time.Millisecond} + _, err := client.Get("http://" + addrOld + prefixMetaData) + assert.Error(t, err, "stale listener must be closed after rebind") + + m.shutdown() +} + func TestTapResponder_StartRejectsMissAndError(t *testing.T) { var addrs sync.Map listen := loopbackTapListen(&addrs) diff --git a/spinifex/handlers/sysinstance/launcher.go b/spinifex/handlers/sysinstance/launcher.go index 15d7cbef5..2a6f1e03f 100644 --- a/spinifex/handlers/sysinstance/launcher.go +++ b/spinifex/handlers/sysinstance/launcher.go @@ -1,6 +1,7 @@ // Package sysinstance holds the boot-agnostic contract for system-managed VMs. // Hidden from DescribeInstances; supports two boot styles: BootDirect (fw_cfg microVM) -// and BootAMI (AMI + cloud-init). Types live here to avoid an eks→elbv2 import cycle. +// and BootAMI (AMI booting from the Ec2 IMDS datasource). Types live here to avoid an +// eks→elbv2 import cycle. package sysinstance import "errors" @@ -15,7 +16,8 @@ type BootMode string const ( // BootDirect is the direct-boot microVM path (fw_cfg-delivered config). BootDirect BootMode = "direct" - // BootAMI is the AMI + cloud-init path (root volume + seed ISO). + // BootAMI boots a root volume cloned from an AMI off the Ec2 IMDS datasource; + // its mgmt NIC is configured from a fw_cfg netcfg blob. BootAMI BootMode = "ami" ) diff --git a/spinifex/lbagent/agent.go b/spinifex/lbagent/agent.go index b31605e8c..8a1cf511e 100644 --- a/spinifex/lbagent/agent.go +++ b/spinifex/lbagent/agent.go @@ -93,7 +93,7 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error) return nil, fmt.Errorf("region is required") } - // Use system CA trust store (CA cert injected via cloud-init ca_certs). + // Use system CA trust store (deployment CA provisioned via fw_cfg on the microvm). client := &http.Client{ Timeout: 10 * time.Second, Transport: &http.Transport{ diff --git a/spinifex/network/host/imds_attach.go b/spinifex/network/host/imds_attach.go index d57de2bd4..c4cf8843d 100644 --- a/spinifex/network/host/imds_attach.go +++ b/spinifex/network/host/imds_attach.go @@ -65,7 +65,8 @@ func imdsDetachSpec(eniID string) IMDSTapDatapath { // ListIMDSTaps advertises the tap off the patch and vpcd binds the endpoint, so the // endpoint must exist whenever the patch does, else vpcd binds a missing device every // reconcile forever. The flows and reply routing only serve, so theirs wrap -// ErrIMDSServingDegraded — the endpoint is already bound, so the caller logs on. +// ErrIMDSServingDegraded to mark that connectivity is intact — but the launch caller +// aborts on it all the same, since guest bootstrap now depends on IMDS. func installIMDSDatapath(ctx context.Context, r Runner, d IMDSTapDatapath) error { if err := d.validate(); err != nil { return err diff --git a/spinifex/network/host/imds_datapath.go b/spinifex/network/host/imds_datapath.go index a1ecc2adf..ddfd3a16f 100644 --- a/spinifex/network/host/imds_datapath.go +++ b/spinifex/network/host/imds_datapath.go @@ -6,7 +6,6 @@ import ( "fmt" "hash/fnv" "log/slog" - "strings" "github.com/mulgadc/spinifex/spinifex/utils" ) @@ -191,9 +190,11 @@ func ensureIMDSEndpoint(ctx context.Context, r Runner, d IMDSTapDatapath) error if _, err := r.Run(ctx, "ip", "link", "set", d.Endpoint, "up"); err != nil { return fmt.Errorf("bring up IMDS endpoint %s: %w", d.Endpoint, err) } + // `replace` is idempotent — adds the /32 if absent, no-op if the endpoint + // already owns it (a recovery/stop re-attach reuses a surviving endpoint). + // `add` errored on the duplicate, and its kernel message varies by version. for _, addr := range imdsCaptureAddrs { - out, err := r.Run(ctx, "ip", "addr", "add", addr+"/32", "dev", d.Endpoint) - if err != nil && !strings.Contains(string(out), "File exists") { + if _, err := r.Run(ctx, "ip", "addr", "replace", addr+"/32", "dev", d.Endpoint); err != nil { return fmt.Errorf("add %s to IMDS endpoint %s: %w", addr, d.Endpoint, err) } } diff --git a/spinifex/network/host/imds_datapath_test.go b/spinifex/network/host/imds_datapath_test.go index 4ca5d1e1f..310facc40 100644 --- a/spinifex/network/host/imds_datapath_test.go +++ b/spinifex/network/host/imds_datapath_test.go @@ -128,8 +128,8 @@ func TestInstallTapDatapath(t *testing.T) { "ovs-vsctl --may-exist add-port " + IMDSBridge + " " + d.Endpoint + " -- set Interface " + d.Endpoint + " type=internal", "ip link set " + d.Endpoint + " address " + d.EndpointMAC, "ip link set " + d.Endpoint + " up", - "ip addr add " + imdsMetaAddr + "/32 dev " + d.Endpoint, - "ip addr add " + imdsDNSAddr + "/32 dev " + d.Endpoint, + "ip addr replace " + imdsMetaAddr + "/32 dev " + d.Endpoint, + "ip addr replace " + imdsDNSAddr + "/32 dev " + d.Endpoint, "sysctl -qw net.ipv4.conf." + d.Endpoint + ".rp_filter=0", "sysctl -qw net.ipv4.conf." + d.Endpoint + ".accept_local=1", // Flows are not cleared here: installIMDSDatapath clears the shared cookie @@ -153,16 +153,26 @@ func TestInstallTapDatapath(t *testing.T) { } } -func TestInstallTapDatapathToleratesExistingAddr(t *testing.T) { +// TestInstallTapDatapathReattachIsIdempotent guards the recovery/stop re-attach +// path: captured addresses are added with `ip addr replace`, a no-op when the +// surviving endpoint already owns them. `ip addr add` errored on the duplicate +// ("Address already assigned"), aborting the launch. +func TestInstallTapDatapathReattachIsIdempotent(t *testing.T) { s := newStubRunner() s.expect("ovs-vsctl", nil, nil) - s.expect("ip link set", nil, nil) - s.expect("ip addr add", []byte("RTNETLINK answers: File exists"), errors.New("exit status 2")) + s.expect("ip", nil, nil) s.expect("sysctl", nil, nil) s.expect("ovs-ofctl", nil, nil) - if err := InstallTapDatapath(context.Background(), s, testDatapath()); err != nil { - t.Fatalf("File exists on addr add must be tolerated, got: %v", err) + d := testDatapath() + if err := InstallTapDatapath(context.Background(), s, d); err != nil { + t.Fatalf("InstallTapDatapath: %v", err) + } + if !s.called("ip addr replace " + imdsMetaAddr + "/32 dev " + d.Endpoint) { + t.Errorf("captured addr must be added with idempotent `ip addr replace`; calls: %v", s.calls) + } + if s.called("ip addr add ") { + t.Errorf("`ip addr add` errors on a duplicate re-attach; use `ip addr replace`; calls: %v", s.calls) } } diff --git a/spinifex/services/viperblockd/viperblockd.go b/spinifex/services/viperblockd/viperblockd.go index f068fb9ff..4dc25929f 100644 --- a/spinifex/services/viperblockd/viperblockd.go +++ b/spinifex/services/viperblockd/viperblockd.go @@ -235,11 +235,11 @@ func openVolumeVB(cfg *Config, volumeName string) (*viperblock.VB, error) { return vb, nil } -// isAuxVolume reports whether a volume is an -efi/-cloudinit auxiliary volume. -// Auxiliary volumes are recreated on launch and carry no durable guest data, so -// they never need sealing to predastore. +// isAuxVolume reports whether a volume is an -efi auxiliary volume. Auxiliary +// volumes are recreated on launch and carry no durable guest data, so they +// never need sealing to predastore. func isAuxVolume(volumeName string) bool { - return strings.HasSuffix(volumeName, "-efi") || strings.HasSuffix(volumeName, "-cloudinit") + return strings.HasSuffix(volumeName, "-efi") } // volumeNeedsSeal reports whether an unmounted volume must be sealed to @@ -713,10 +713,10 @@ func launchService(cfg *Config) (err error) { vb, err := viperblock.New(&vbconfig, "s3", s3cfg) - // Enable 128MB cache for main volumes, disable for cloudinit/efi (small, rarely read) + // Enable 128MB cache for main volumes, disable for efi (small, rarely read) // This cacheSize is passed to nbdkit plugin (separate viperblock instance) var nbdCacheSize int - if strings.HasSuffix(ebsRequest.Name, "-cloudinit") || strings.HasSuffix(ebsRequest.Name, "-efi") { + if strings.HasSuffix(ebsRequest.Name, "-efi") { slog.Info("Disabling cache for auxiliary volume", "volume", ebsRequest.Name) if err := vb.SetCacheSize(0, 0); err != nil { slog.Error("Failed to set cache size", "err", err) diff --git a/spinifex/types/ebs.go b/spinifex/types/ebs.go index 8ad217f01..dbe4587c0 100644 --- a/spinifex/types/ebs.go +++ b/spinifex/types/ebs.go @@ -15,7 +15,6 @@ type EBSRequest struct { VolType string `json:"VolType"` Boot bool `json:"Boot"` EFI bool `json:"EFI"` - CloudInit bool `json:"CloudInit"` DeleteOnTermination bool `json:"DeleteOnTermination"` NBDURI string `json:"NBDURI"` // NBD URI - socket path (nbd:unix:/path.sock) or TCP (nbd://host:port) DeviceName string `json:"DeviceName"` // AWS API device name (e.g. /dev/sdf) for hot-plugged volumes diff --git a/spinifex/utils/images.go b/spinifex/utils/images.go index 0a1dcfb3a..e2efb993a 100644 --- a/spinifex/utils/images.go +++ b/spinifex/utils/images.go @@ -369,9 +369,9 @@ var AvailableImages = map[string]Images{ Version: "10", Arch: "x86_64", Platform: "Linux/UNIX", - CreatedAt: time.Date(2025, 11, 16, 0, 0, 0, 0, time.UTC), - URL: "https://dl.rockylinux.org/pub/rocky/10/images/x86_64/Rocky-10-GenericCloud-Base-10.1-20251116.0.x86_64.qcow2", - Checksum: "https://dl.rockylinux.org/pub/rocky/10/images/x86_64/Rocky-10-GenericCloud-Base-10.1-20251116.0.x86_64.qcow2.CHECKSUM", + CreatedAt: time.Date(2026, 5, 25, 0, 0, 0, 0, time.UTC), + URL: "https://dl.rockylinux.org/pub/rocky/10/images/x86_64/Rocky-10-GenericCloud-Base-10.2-20260525.0.x86_64.qcow2", + Checksum: "https://dl.rockylinux.org/pub/rocky/10/images/x86_64/Rocky-10-GenericCloud-Base-10.2-20260525.0.x86_64.qcow2.CHECKSUM", ChecksumType: "sha256", BootMode: "uefi", }, @@ -383,9 +383,9 @@ var AvailableImages = map[string]Images{ Version: "10", Arch: "arm64", Platform: "Linux/UNIX", - CreatedAt: time.Date(2025, 11, 16, 0, 0, 0, 0, time.UTC), - URL: "https://dl.rockylinux.org/pub/rocky/10/images/aarch64/Rocky-10-GenericCloud-Base-10.1-20251116.0.aarch64.qcow2", - Checksum: "https://dl.rockylinux.org/pub/rocky/10/images/aarch64/Rocky-10-GenericCloud-Base-10.1-20251116.0.aarch64.qcow2.CHECKSUM", + CreatedAt: time.Date(2026, 5, 25, 0, 0, 0, 0, time.UTC), + URL: "https://dl.rockylinux.org/pub/rocky/10/images/aarch64/Rocky-10-GenericCloud-Base-10.2-20260525.0.aarch64.qcow2", + Checksum: "https://dl.rockylinux.org/pub/rocky/10/images/aarch64/Rocky-10-GenericCloud-Base-10.2-20260525.0.aarch64.qcow2.CHECKSUM", ChecksumType: "sha256", BootMode: "uefi", }, diff --git a/spinifex/vm/device_map.go b/spinifex/vm/device_map.go index 29c1dba55..b659878a5 100644 --- a/spinifex/vm/device_map.go +++ b/spinifex/vm/device_map.go @@ -154,8 +154,6 @@ func (m *Manager) UpdateGuestDeviceNames(instance *VM) { var qemuID string if req.Boot { qemuID = "os" - } else if req.CloudInit { - qemuID = "cloudinit" } else { qemuID = fmt.Sprintf("vdisk-%s", req.Name) } diff --git a/spinifex/vm/errors.go b/spinifex/vm/errors.go index 6eb5de11d..f16ab8cca 100644 --- a/spinifex/vm/errors.go +++ b/spinifex/vm/errors.go @@ -20,7 +20,7 @@ var ErrAttachmentLimitExceeded = errors.New("attachment limit exceeded") var ErrVolumeNotAttached = errors.New("volume not attached to instance") // ErrVolumeNotDetachable is returned by DetachVolume when the target -// volume is a boot, EFI, or cloud-init volume that cannot be hot-unplugged. +// volume is a boot or EFI volume that cannot be hot-unplugged. var ErrVolumeNotDetachable = errors.New("volume cannot be detached") // ErrVolumeDeviceMismatch is returned by DetachVolume when the caller diff --git a/spinifex/vm/instance_cleaner.go b/spinifex/vm/instance_cleaner.go index aa2394d1b..f5f818488 100644 --- a/spinifex/vm/instance_cleaner.go +++ b/spinifex/vm/instance_cleaner.go @@ -4,8 +4,8 @@ package vm // public IP, ENI, placement group) that the manager does not own. Invoked from // Stop/Terminate after QEMU shuts down; methods are best-effort and self-logging. type InstanceCleaner interface { - // DeleteVolumes deletes EFI / cloud-init internal volumes via ebs.delete - // and user volumes flagged DeleteOnTermination via the volume service. + // DeleteVolumes deletes the EFI internal volume via ebs.delete and user + // volumes flagged DeleteOnTermination via the volume service. // Called only on Terminate, never on Stop. Returns the first delete error. DeleteVolumes(v *VM) error diff --git a/spinifex/vm/lifecycle.go b/spinifex/vm/lifecycle.go index 8e6e2edc7..7eacea572 100644 --- a/spinifex/vm/lifecycle.go +++ b/spinifex/vm/lifecycle.go @@ -2,6 +2,8 @@ package vm import ( "bufio" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -328,6 +330,9 @@ func (m *Manager) startQEMU(instance *VM) error { Value: fmt.Sprintf("tap,id=mgmt0,ifname=%s,script=no,downscript=no", mgmtTap), }) instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "mgmt0", instance.MgmtMAC)) + if err := m.appendSystemNetcfgFwCfg(instance); err != nil { + return fmt.Errorf("attach system netcfg: %w", err) + } slog.Info("Management NIC configured", "tap", mgmtTap, "mac", instance.MgmtMAC, "ip", instance.MgmtIP, "instanceId", instance.ID) } @@ -724,6 +729,48 @@ func reconnectQMP(q *qmp.QMPClient, instanceID string) error { // EBS hot-plug, matching the /dev/sd[f-p] range. Cannot grow without QEMU restart. const EBSHotPlugSlotCount = 11 +// appendSystemNetcfgFwCfg attaches an fw_cfg netcfg blob describing a multi-NIC +// BootAMI system VM's interfaces so the guest brings them up deterministically +// by MAC. The EKS control plane is the case: cloud-init on a stock Alpine guest +// cannot reliably pick the right NIC out of two, so it must be told which is +// which. The primary data ENI is marked DHCP (OVN serves it; it carries the +// default route, and bringing it up before cloud-init's network stage is what +// lets the Ec2 datasource reach IMDS — without it cloud-init brings up the mgmt +// NIC instead and falls to DataSourceNone). mgmt0 lives on br-mgmt with no DHCP, +// so its static address is delivered here; it is never the default route. The +// blob key format matches daemon.buildNetcfgBlob and build/microvm/init.sh. +// No-op without a mgmt NIC, so single-NIC guests are untouched — cloud-init +// brings their one NIC up. +func (m *Manager) appendSystemNetcfgFwCfg(instance *VM) error { + if instance.MgmtMAC == "" || instance.MgmtIP == "" { + return nil + } + var b strings.Builder + n := 0 + // Primary data ENI: DHCP (OVN-served) + default route. + if instance.ENIMac != "" { + fmt.Fprintf(&b, "NIC%d_MAC=%s\nNIC%d_DHCP=1\nNIC%d_DEFAULT=1\n", n, instance.ENIMac, n, n) + n++ + } + // Management NIC: static, off br-mgmt, never the default route. + fmt.Fprintf(&b, "NIC%d_MAC=%s\nNIC%d_CIDR=%s/24\nNIC%d_DEFAULT=0\n", n, instance.MgmtMAC, n, instance.MgmtIP, n) + + path := filepath.Join(utils.RuntimeDir(), fmt.Sprintf("fwcfg-%s-netcfg.tmp", instance.ID)) + if err := os.WriteFile(path, []byte(b.String()), 0o600); err != nil { + return fmt.Errorf("write system netcfg blob: %w", err) + } + instance.Config.FwCfg = append(instance.Config.FwCfg, FwCfgEntry{Name: "opt/spinifex/netcfg", File: path}) + return nil +} + +// ec2SMBIOSUUID derives a deterministic, "ec2"-prefixed system UUID from the +// instance ID so cloud-init's Ec2 datasource activates (stable across reboot). +func ec2SMBIOSUUID(instanceID string) string { + sum := sha256.Sum256([]byte("ec2-smbios:" + instanceID)) + h := "ec2" + hex.EncodeToString(sum[:])[3:32] + return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:32]) +} + // buildBaseVMConfig creates a Config with base QEMU settings and two pre-allocated // PCIe root-port pools: hotplug-ebs{1..N} for EBS (/dev/sd[f-p]) and // hotplug-eni{1..M} for ENI hot-plug. bootMode "uefi"/"uefi-preferred" sets UseUEFI. @@ -742,6 +789,11 @@ func buildBaseVMConfig(instanceID, instanceType, pidFile, consoleLogPath, serial Architecture: architecture, InstanceType: instanceType, UseUEFI: bootMode == "uefi" || bootMode == "uefi-preferred", + + // Present EC2-shaped DMI so stock cloud-init activates the Ec2 datasource. + SMBIOSUUID: ec2SMBIOSUUID(instanceID), + SMBIOSManufacturer: "Amazon EC2", + SMBIOSAssetTag: "Amazon EC2", } for i := 1; i <= EBSHotPlugSlotCount; i++ { @@ -804,13 +856,6 @@ func buildDrives(requests []types.EBSRequest, cpuCount int, machineType string) devices = append(devices, BlkDevice(machineType, drive.ID, iothreadID, cpuCount, 1)) } - if v.CloudInit { - drive.Format = "raw" - drive.If = "virtio" - drive.Media = "cdrom" - drive.ID = "cloudinit" - } - if v.EFI { drive.Format = "raw" drive.If = "pflash" diff --git a/spinifex/vm/lifecycle_test.go b/spinifex/vm/lifecycle_test.go index 61383cfc6..dc070c5c9 100644 --- a/spinifex/vm/lifecycle_test.go +++ b/spinifex/vm/lifecycle_test.go @@ -127,16 +127,6 @@ func TestBuildDrives(t *testing.T) { {Value: "virtio-blk-pci,drive=os,iothread=ioth-os,num-queues=4,bootindex=1"}, }, }, - { - name: "cloud-init volume", - requests: []types.EBSRequest{ - {Name: "vol-ci", NBDURI: "nbd:unix:/tmp/ci.sock", CloudInit: true}, - }, - cpuCount: 2, - wantDrives: []Drive{ - {File: "nbd:unix:/tmp/ci.sock", Format: "raw", If: "virtio", Media: "cdrom", ID: "cloudinit"}, - }, - }, { name: "EFI volume emits pflash unit=1", requests: []types.EBSRequest{ @@ -164,16 +154,14 @@ func TestBuildDrives(t *testing.T) { wantErr: "NBDURI not set for volume vol-efi-bad", }, { - name: "mixed boot + cloud-init + EFI", + name: "mixed boot + EFI", requests: []types.EBSRequest{ {Name: "vol-boot", NBDURI: "nbd:unix:/tmp/boot.sock", Boot: true}, - {Name: "vol-ci", NBDURI: "nbd:unix:/tmp/ci.sock", CloudInit: true}, {Name: "vol-efi", NBDURI: "nbd:unix:/tmp/efi.sock", EFI: true}, }, cpuCount: 4, wantDrives: []Drive{ {File: "nbd:unix:/tmp/boot.sock", Format: "raw", If: "none", Media: "disk", ID: "os", Cache: "none"}, - {File: "nbd:unix:/tmp/ci.sock", Format: "raw", If: "virtio", Media: "cdrom", ID: "cloudinit"}, {File: "nbd:unix:/tmp/efi.sock", Format: "raw", If: "pflash", Unit: 1}, }, wantIOThreads: []IOThread{{ID: "ioth-os"}}, diff --git a/spinifex/vm/mgmt_netcfg_test.go b/spinifex/vm/mgmt_netcfg_test.go new file mode 100644 index 000000000..29ec105d2 --- /dev/null +++ b/spinifex/vm/mgmt_netcfg_test.go @@ -0,0 +1,41 @@ +package vm + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAppendSystemNetcfgFwCfg(t *testing.T) { + m := &Manager{} + + t.Run("emits a data (DHCP) + mgmt (static) NIC blob for a multi-NIC system VM", func(t *testing.T) { + t.Setenv("XDG_RUNTIME_DIR", t.TempDir()) + inst := &VM{ID: "i-sys01", ENIMac: "02:3c:8f:54:bd:c9", MgmtMAC: "02:aa:bb:cc:dd:ee", MgmtIP: "10.20.0.5"} + + require.NoError(t, m.appendSystemNetcfgFwCfg(inst)) + + require.Len(t, inst.Config.FwCfg, 1) + entry := inst.Config.FwCfg[0] + require.Equal(t, "opt/spinifex/netcfg", entry.Name) + + data, err := os.ReadFile(entry.File) + require.NoError(t, err) + // Format must match build/microvm/init.sh + the eks-node mulga-mgmt-net + // consumer: the data ENI is DHCP + default route, mgmt0 is static and + // never the default route. + require.Equal(t, + "NIC0_MAC=02:3c:8f:54:bd:c9\nNIC0_DHCP=1\nNIC0_DEFAULT=1\n"+ + "NIC1_MAC=02:aa:bb:cc:dd:ee\nNIC1_CIDR=10.20.0.5/24\nNIC1_DEFAULT=0\n", + string(data)) + }) + + t.Run("no mgmt NIC is a no-op (single-NIC guests use cloud-init/IMDS)", func(t *testing.T) { + t.Setenv("XDG_RUNTIME_DIR", t.TempDir()) + inst := &VM{ID: "i-nomgmt", ENIMac: "02:3c:8f:54:bd:c9"} + + require.NoError(t, m.appendSystemNetcfgFwCfg(inst)) + require.Empty(t, inst.Config.FwCfg) + }) +} diff --git a/spinifex/vm/network_helpers.go b/spinifex/vm/network_helpers.go index a7fd36887..9a769f162 100644 --- a/spinifex/vm/network_helpers.go +++ b/spinifex/vm/network_helpers.go @@ -1,7 +1,6 @@ package vm import ( - "errors" "fmt" "log/slog" "strings" @@ -93,9 +92,12 @@ func GenerateMgmtMAC(instanceID string) string { // attachPrimaryIMDSDatapath installs the per-tap IMDS datapath for the instance's // primary ENI once its tap is up, before QEMU starts the guest. Only the primary -// ENI serves IMDS. A serving-only failure (ErrIMDSServingDegraded) leaves the guest -// connected and is logged-and-continued; a connectivity-critical failure would -// strand the tap on br-imds, so it is rolled back to br-int. +// ENI serves IMDS. Any failure is launch-fatal: guest bootstrap (SSH key, user-data, +// network, hostname) now comes from IMDS, so a guest without it would boot +// unconfigured while reporting success. A serving-only failure +// (ErrIMDSServingDegraded) and a connectivity-critical failure are treated alike — +// there is no roll back to br-int, since a rolled-back tap is equally unreachable for +// IMDS. The error surfaces to RunInstances. func (m *Manager) attachPrimaryIMDSDatapath(instance *VM) error { if m.deps.NetworkPlumber == nil || instance.ENIId == "" { return nil @@ -109,45 +111,11 @@ func (m *Manager) attachPrimaryIMDSDatapath(instance *VM) error { "instance", instance.ID, "eni", instance.ENIId) return nil } - err := m.deps.NetworkPlumber.AttachIMDSDatapath(instance.ENIId, instance.ENIMac, subnetID) - if err == nil { - return nil - } - if errors.Is(err, ErrIMDSServingDegraded) { - // Connectivity is intact (the patch + forward flows installed); only the - // IMDS demux/egress/reply stage failed. The guest keeps full VPC - // connectivity, so log and continue — boot is not gated on IMDS readiness. - slog.Error("IMDS: per-tap serving install failed; guest keeps connectivity but IMDS is unavailable", + if err := m.deps.NetworkPlumber.AttachIMDSDatapath(instance.ENIId, instance.ENIMac, subnetID); err != nil { + slog.Error("IMDS: per-tap datapath install failed; aborting launch", "instance", instance.ID, "eni", instance.ENIId, "err", err) - return nil + return fmt.Errorf("install IMDS datapath for primary ENI %s: %w", instance.ENIId, err) } - // Connectivity-critical failure: the tap is stranded on br-imds with no path to - // OVN, so the guest would boot black-holed. Roll it back to br-int — losing IMDS - // but restoring connectivity — and fail the launch only if even that fails. - slog.Error("IMDS: per-tap connectivity install failed; rolling primary tap back to br-int", - "instance", instance.ID, "eni", instance.ENIId, "err", err) - return m.rollbackPrimaryTapToBrInt(instance) -} - -// rollbackPrimaryTapToBrInt moves the primary tap off br-imds back onto br-int after -// a connectivity-critical IMDS failure, restoring the guest's L2 path to OVN at the -// cost of IMDS. It detaches any partial datapath (freeing the iface-id), removes the -// tap from br-imds, then re-plumbs it on br-int. Only the re-plumb failing is fatal. -func (m *Manager) rollbackPrimaryTapToBrInt(instance *VM) error { - if err := m.deps.NetworkPlumber.DetachIMDSDatapath(instance.ENIId); err != nil { - slog.Warn("IMDS: rollback detach failed (continuing)", - "instance", instance.ID, "eni", instance.ENIId, "err", err) - } - tapName := TapDeviceName(instance.ENIId) - if err := m.deps.NetworkPlumber.CleanupTap(tapName); err != nil { - slog.Warn("IMDS: rollback tap cleanup failed (continuing)", - "instance", instance.ID, "tap", tapName, "err", err) - } - if err := m.deps.NetworkPlumber.SetupTap(VPCTapSpec(instance.ENIId, instance.ENIMac)); err != nil { - return fmt.Errorf("roll primary tap %s back to br-int: %w", tapName, err) - } - slog.Warn("IMDS: primary tap rolled back to br-int; guest connected, IMDS unavailable", - "instance", instance.ID, "tap", tapName, "eni", instance.ENIId) return nil } @@ -167,8 +135,7 @@ func (m *Manager) detachPrimaryIMDSDatapath(instance *VM) { // setupExtraENINICs creates tap devices on br-int and appends matching QEMU // virtio-net device entries to instance.Config for each additional ENI a // system VM spans. The primary ENI is handled separately by the launch -// caller. Cloud-init brings the guest interfaces up via per-MAC DHCP blocks -// written by generateNetworkConfig. +// caller. The guest brings each interface up via per-MAC DHCP. func (m *Manager) setupExtraENINICs(instance *VM) error { if m.deps.NetworkPlumber == nil { return nil diff --git a/spinifex/vm/network_helpers_test.go b/spinifex/vm/network_helpers_test.go index 59e580c50..6645ad76b 100644 --- a/spinifex/vm/network_helpers_test.go +++ b/spinifex/vm/network_helpers_test.go @@ -195,79 +195,58 @@ func TestAttachPrimaryIMDSDatapath_NilPlumber_NoOp(t *testing.T) { } } -func TestAttachPrimaryIMDSDatapath_ServingDegradedIsNonFatal(t *testing.T) { - // Serving-only failure: connectivity (patch + forward flows) is intact, only - // the IMDS demux/reply stage failed. The guest stays connected, so the launch - // proceeds and the tap is NOT rolled back to br-int. +func TestAttachPrimaryIMDSDatapath_ServingDegradedIsFatal(t *testing.T) { + // Serving-only failure: connectivity (patch + forward flows) is intact, only the + // IMDS demux/reply stage failed. Guest bootstrap now comes from IMDS, so an + // unreachable IMDS would boot a silently-unconfigured guest — the launch must + // fail. No roll back to br-int (a rolled-back tap is equally unreachable for IMDS). plumber := &fakeNetworkPlumber{ imdsAttachErr: fmt.Errorf("%w: install endpoint: ovsdb busy", ErrIMDSServingDegraded), } m := NewManagerWithDeps(Deps{NetworkPlumber: plumber}) - if err := m.attachPrimaryIMDSDatapath(&VM{ - ID: "i-serving-degraded", - ENIId: "eni-abc123", - ENIMac: "02:00:00:aa:bb:cc", - Instance: &ec2.Instance{SubnetId: aws.String("subnet-xyz")}, - }); err != nil { - t.Fatalf("serving-degraded attach must not fail the launch, got %v", err) - } - if len(plumber.imdsAttachCalls) != 1 { - t.Errorf("expected the attach to be attempted once, got %d calls", len(plumber.imdsAttachCalls)) - } - if len(plumber.imdsDetachCalls) != 0 || len(plumber.cleanupCalls) != 0 || len(plumber.setupCalls) != 0 { - t.Errorf("serving-degraded must not roll back: detach=%d cleanup=%d setup=%d", - len(plumber.imdsDetachCalls), len(plumber.cleanupCalls), len(plumber.setupCalls)) - } -} - -func TestAttachPrimaryIMDSDatapath_ConnectivityFailureRollsBack(t *testing.T) { - // Connectivity-critical failure: the primary tap is stranded on the secure - // br-imds. The tap must be rolled back to br-int (detach partial datapath, - // remove the stranded tap, re-plumb on br-int) so the guest is not black-holed. - plumber := &fakeNetworkPlumber{imdsAttachErr: errors.New("ovsdb lock held")} - m := NewManagerWithDeps(Deps{NetworkPlumber: plumber}) - instance := &VM{ - ID: "i-conn-err", - ENIId: "eni-abc123", - ENIMac: "02:00:00:aa:bb:cc", - Instance: &ec2.Instance{SubnetId: aws.String("subnet-xyz")}, - } - if err := m.attachPrimaryIMDSDatapath(instance); err != nil { - t.Fatalf("a successful rollback must not fail the launch, got %v", err) - } - if len(plumber.imdsDetachCalls) != 1 { - t.Errorf("expected one rollback detach, got %d", len(plumber.imdsDetachCalls)) - } - wantTap := TapDeviceName(instance.ENIId) - if !reflect.DeepEqual(plumber.cleanupCalls, []string{wantTap}) { - t.Errorf("cleanupCalls = %v, want [%q]", plumber.cleanupCalls, wantTap) - } - // The tap must be re-plumbed onto br-int with the OVN binding, not br-imds. - wantSpec := VPCTapSpec(instance.ENIId, instance.ENIMac) - if !reflect.DeepEqual(plumber.setupCalls, []TapSpec{wantSpec}) { - t.Errorf("setupCalls = %+v, want [%+v]", plumber.setupCalls, wantSpec) - } -} - -func TestAttachPrimaryIMDSDatapath_RollbackFailureFailsLaunch(t *testing.T) { - // If even the roll-back onto br-int fails, the guest has no connected path at - // all, so the launch must fail rather than boot a black-holed guest. - plumber := &fakeNetworkPlumber{ - imdsAttachErr: errors.New("ovsdb lock held"), - setupErr: errors.New("br-int add-port failed"), - } - m := NewManagerWithDeps(Deps{NetworkPlumber: plumber}) err := m.attachPrimaryIMDSDatapath(&VM{ - ID: "i-rollback-err", + ID: "i-serving-degraded", ENIId: "eni-abc123", ENIMac: "02:00:00:aa:bb:cc", Instance: &ec2.Instance{SubnetId: aws.String("subnet-xyz")}, }) if err == nil { - t.Fatal("expected a failed rollback to fail the launch, got nil") + t.Fatal("serving-degraded attach must fail the launch, got nil") } - if !strings.Contains(err.Error(), "br-int") { - t.Errorf("error = %v, want it to mention the failing br-int re-plumb", err) + if !errors.Is(err, ErrIMDSServingDegraded) { + t.Errorf("error = %v, want it to wrap ErrIMDSServingDegraded", err) + } + if len(plumber.imdsAttachCalls) != 1 { + t.Errorf("expected the attach to be attempted once, got %d calls", len(plumber.imdsAttachCalls)) + } + if len(plumber.imdsDetachCalls) != 0 || len(plumber.cleanupCalls) != 0 || len(plumber.setupCalls) != 0 { + t.Errorf("must not roll back: detach=%d cleanup=%d setup=%d", + len(plumber.imdsDetachCalls), len(plumber.cleanupCalls), len(plumber.setupCalls)) + } +} + +func TestAttachPrimaryIMDSDatapath_ConnectivityFailureIsFatal(t *testing.T) { + // Connectivity-critical failure: the primary tap is stranded on br-imds. The + // launch must fail — there is no roll back to br-int, since a rolled-back tap is + // equally unreachable for IMDS and a clear error beats a half-working box. + plumber := &fakeNetworkPlumber{imdsAttachErr: errors.New("ovsdb lock held")} + m := NewManagerWithDeps(Deps{NetworkPlumber: plumber}) + err := m.attachPrimaryIMDSDatapath(&VM{ + ID: "i-conn-err", + ENIId: "eni-abc123", + ENIMac: "02:00:00:aa:bb:cc", + Instance: &ec2.Instance{SubnetId: aws.String("subnet-xyz")}, + }) + if err == nil { + t.Fatal("connectivity-critical attach must fail the launch, got nil") + } + if !strings.Contains(err.Error(), "ovsdb lock held") { + t.Errorf("error = %v, want it to wrap the attach failure", err) + } + // No rollback: the tap is not detached, cleaned up, or re-plumbed onto br-int. + if len(plumber.imdsDetachCalls) != 0 || len(plumber.cleanupCalls) != 0 || len(plumber.setupCalls) != 0 { + t.Errorf("must not roll back: detach=%d cleanup=%d setup=%d", + len(plumber.imdsDetachCalls), len(plumber.cleanupCalls), len(plumber.setupCalls)) } } diff --git a/spinifex/vm/network_plumber.go b/spinifex/vm/network_plumber.go index f56aefcda..9700dcc40 100644 --- a/spinifex/vm/network_plumber.go +++ b/spinifex/vm/network_plumber.go @@ -4,8 +4,11 @@ import "errors" // ErrIMDSServingDegraded marks an AttachIMDSDatapath failure after the // connectivity-critical datapath (patch + forward flows) was installed — only the -// IMDS demux/egress/reply stage failed, so the guest keeps full VPC connectivity and -// the caller logs and continues. Any other error means connectivity was not set up. +// IMDS demux/egress/reply stage failed, so the guest keeps VPC connectivity but +// cannot reach IMDS. Since guest bootstrap (SSH key, user-data, network, hostname) +// now comes from IMDS, this is launch-fatal: the caller aborts rather than boot a +// guest that would silently come up unconfigured. Any other error means connectivity +// was not set up and is equally fatal. var ErrIMDSServingDegraded = errors.New("imds: per-tap serving install failed (guest connectivity intact)") // NetworkPlumber handles tap device and OVS bridge operations. Defined in vm diff --git a/spinifex/vm/smbios_test.go b/spinifex/vm/smbios_test.go new file mode 100644 index 000000000..da56d52ed --- /dev/null +++ b/spinifex/vm/smbios_test.go @@ -0,0 +1,75 @@ +package vm + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExecute_EmitsEC2SMBIOS(t *testing.T) { + uuid := ec2SMBIOSUUID("i-0123456789abcdef0") + cfg := Config{ + CPUCount: 2, + Memory: 1024, + Architecture: "x86_64", + Drives: []Drive{{File: "disk.img", Format: "qcow2"}}, + SMBIOSUUID: uuid, + SMBIOSManufacturer: "Amazon EC2", + SMBIOSAssetTag: "Amazon EC2", + } + + cmd, err := cfg.Execute() + require.NoError(t, err) + args := cmd.Args[1:] + + assert.Equal(t, uuid, argValue(args, "-uuid")) + + // Both -smbios entries must be present: type=1 manufacturer/serial, type=3 asset. + smbios := allArgValues(args, "-smbios") + assert.Contains(t, smbios, "type=1,manufacturer=Amazon EC2,serial="+uuid) + assert.Contains(t, smbios, "type=3,asset=Amazon EC2") +} + +func TestExecute_NoSMBIOSWhenUnset(t *testing.T) { + cfg := Config{ + CPUCount: 2, + Memory: 1024, + Architecture: "x86_64", + Drives: []Drive{{File: "disk.img", Format: "qcow2"}}, + } + + cmd, err := cfg.Execute() + require.NoError(t, err) + args := cmd.Args[1:] + + assert.False(t, argExists(args, "-uuid"), "no -uuid expected when SMBIOSUUID unset") + assert.False(t, argExists(args, "-smbios"), "no -smbios expected when SMBIOS fields unset") +} + +func TestEC2SMBIOSUUID(t *testing.T) { + id := "i-0123456789abcdef0" + u := ec2SMBIOSUUID(id) + + // cloud-init's identify_aws keys on an "ec2"-prefixed product_uuid. + assert.True(t, strings.HasPrefix(u, "ec2"), "uuid must be ec2-prefixed: %s", u) + + // Canonical 8-4-4-4-12 UUID shape. + assert.Len(t, u, 36) + parts := strings.Split(u, "-") + require.Len(t, parts, 5) + assert.Equal(t, []int{8, 4, 4, 4, 12}, []int{len(parts[0]), len(parts[1]), len(parts[2]), len(parts[3]), len(parts[4])}) + + // Deterministic across calls (stable across guest reboot), unique per instance. + assert.Equal(t, u, ec2SMBIOSUUID(id)) + assert.NotEqual(t, u, ec2SMBIOSUUID("i-ffffffffffffffff0")) +} + +func TestBuildBaseVMConfig_SetsEC2SMBIOS(t *testing.T) { + cfg := buildBaseVMConfig("i-0123456789abcdef0", "t3.micro", "/run/pid", "/run/console.log", "/run/serial.sock", "x86_64", "bios", 2, 1024) + + assert.Equal(t, ec2SMBIOSUUID("i-0123456789abcdef0"), cfg.SMBIOSUUID) + assert.Equal(t, "Amazon EC2", cfg.SMBIOSManufacturer) + assert.Equal(t, "Amazon EC2", cfg.SMBIOSAssetTag) +} diff --git a/spinifex/vm/vm.go b/spinifex/vm/vm.go index ac1ccb1ac..beef35ef3 100644 --- a/spinifex/vm/vm.go +++ b/spinifex/vm/vm.go @@ -85,8 +85,8 @@ type VM struct { PublicIPAllocID string `json:"public_ip_alloc_id,omitempty"` PublicIPAssocID string `json:"public_ip_assoc_id,omitempty"` - // DevMAC is the MAC for the dev/hostfwd NIC (DEV_NETWORKING mode). - // Set before cloud-init ISO generation so netplan can suppress its default route. + // DevMAC is the MAC for the dev/hostfwd NIC (DEV_NETWORKING mode), used by + // appendDevHostfwdNIC to give the user-mode NIC a stable address. DevMAC string `json:"dev_mac,omitempty"` // Management NIC for system instance control plane (reaches host via br-mgmt). @@ -220,6 +220,14 @@ type Config struct { // returns an error if no pair is found — no silent SeaBIOS fallback. UseUEFI bool `json:"use_uefi,omitempty"` + // SMBIOSUUID, SMBIOSManufacturer, SMBIOSAssetTag emit EC2-shaped DMI so a stock + // cloud image's cloud-init activates the Ec2 datasource. cloud-init's identify_aws + // keys on /sys/class/dmi/id/product_uuid starting with "ec2", so SMBIOSUUID must + // carry that prefix. + SMBIOSUUID string `json:"smbios_uuid,omitempty"` + SMBIOSManufacturer string `json:"smbios_manufacturer,omitempty"` + SMBIOSAssetTag string `json:"smbios_asset_tag,omitempty"` + KernelImage string `json:"kernel_image,omitempty"` // path to vmlinuz; emits -kernel when set Initrd string `json:"initrd,omitempty"` // path to initramfs; emits -initrd when set KernelCmdline string `json:"kernel_cmdline,omitempty"` // emits -append when set @@ -243,6 +251,18 @@ func (cfg *Config) Execute() (*exec.Cmd, error) { args = append(args, "-qmp", fmt.Sprintf("unix:%s,server,nowait", cfg.QMPSocket)) } + // EC2-shaped SMBIOS so a stock cloud image's cloud-init selects the Ec2 + // datasource. -uuid sets the system UUID cloud-init's identify_aws keys on. + if cfg.SMBIOSUUID != "" { + args = append(args, "-uuid", cfg.SMBIOSUUID) + } + if cfg.SMBIOSManufacturer != "" { + args = append(args, "-smbios", fmt.Sprintf("type=1,manufacturer=%s,serial=%s", cfg.SMBIOSManufacturer, cfg.SMBIOSUUID)) + } + if cfg.SMBIOSAssetTag != "" { + args = append(args, "-smbios", fmt.Sprintf("type=3,asset=%s", cfg.SMBIOSAssetTag)) + } + // Validate native kvm support _, err := os.Stat("/dev/kvm") diff --git a/spinifex/vm/volumes.go b/spinifex/vm/volumes.go index 122a65e45..0fed4e1bd 100644 --- a/spinifex/vm/volumes.go +++ b/spinifex/vm/volumes.go @@ -298,7 +298,7 @@ func (m *Manager) DetachVolume(id, volumeID, device string, force bool) (string, return "", fmt.Errorf("%w: %s", ErrVolumeNotAttached, volumeID) } - if ebsReq.Boot || ebsReq.EFI || ebsReq.CloudInit { + if ebsReq.Boot || ebsReq.EFI { return "", fmt.Errorf("%w: %s", ErrVolumeNotDetachable, volumeID) } diff --git a/spinifex/vm/volumes_test.go b/spinifex/vm/volumes_test.go index cfeadca49..94ff3242f 100644 --- a/spinifex/vm/volumes_test.go +++ b/spinifex/vm/volumes_test.go @@ -306,7 +306,7 @@ func TestDetachVolume_VolumeNotAttached(t *testing.T) { // TestDetachVolume_DeviceGuards covers the four cross-check and undetachable // guards at the head of DetachVolume. Bypassing any of these is destructive: -// detaching a boot/EFI/CloudInit volume kills the instance, and a device-name +// detaching a boot/EFI volume kills the instance, and a device-name // mismatch silently rips out the wrong disk. The mismatch case additionally // pins the error text so an operator-visible diagnostic stays intact. func TestDetachVolume_DeviceGuards(t *testing.T) { @@ -330,12 +330,6 @@ func TestDetachVolume_DeviceGuards(t *testing.T) { requestDevice: "", wantErr: ErrVolumeNotDetachable, }, - { - name: "CloudInit volume rejected", - req: types.EBSRequest{Name: "vol-ci", CloudInit: true, DeviceName: "/dev/sdf"}, - requestDevice: "", - wantErr: ErrVolumeNotDetachable, - }, { name: "device mismatch surfaces expected and actual", req: types.EBSRequest{Name: "vol-1", DeviceName: "/dev/sdf"}, diff --git a/tests/e2e/harness/multinode_guest_ssh.go b/tests/e2e/harness/multinode_guest_ssh.go index a52d7c670..3ba9c15f9 100644 --- a/tests/e2e/harness/multinode_guest_ssh.go +++ b/tests/e2e/harness/multinode_guest_ssh.go @@ -31,8 +31,8 @@ func GuestSSHEndpoint(t *testing.T, c *AWSClient, cluster *Cluster, instanceID s // GuestSSHReady polls SSH into the named instance until a probe command // succeeds. Pemfile is the test-scoped private key written by EnsureKeyPair; -// user defaults to ec2-user (the cloud-init account that spinifex injects -// regardless of distro family — see handlers/ec2/instance/service_impl.go). +// user is the AMI's stock default user (e.g. ubuntu for Ubuntu cloud images), +// which stock cloud-init wires the IMDS launch key to once the seed is retired. func GuestSSHReady(t *testing.T, host string, port int, user, pemfile string, opts ...PollOpt) { t.Helper() cfg := applyOpts(pollCfg{timeout: 60 * time.Second, interval: 1 * time.Second}, opts...) diff --git a/tests/e2e/harness/witness.go b/tests/e2e/harness/witness.go index 71264ce4c..f702bb963 100644 --- a/tests/e2e/harness/witness.go +++ b/tests/e2e/harness/witness.go @@ -53,7 +53,7 @@ type Witness struct { // NewWitness constructs a Witness from environment variables. Required: // AWS_REGION, DDIL_GUEST_SSH_KEY. Optional: DDIL_WITNESS_AMI, // DDIL_WITNESS_INSTANCE_TYPE (defaults to smallest registered type), -// DDIL_WITNESS_KEY_NAME (default spinifex-key), DDIL_GUEST_SSH_USER (default ec2-user). +// DDIL_WITNESS_KEY_NAME (default spinifex-key), DDIL_GUEST_SSH_USER (default ubuntu). func NewWitness(cluster *Cluster, transport SSH) (*Witness, error) { region := os.Getenv("AWS_REGION") if region == "" { @@ -98,7 +98,7 @@ func NewWitness(cluster *Cluster, transport SSH) (*Witness, error) { ssh: transport, hostSigner: hostSigner, guestSigner: guestSigner, - guestUser: envDefault("DDIL_GUEST_SSH_USER", "ec2-user"), + guestUser: envDefault("DDIL_GUEST_SSH_USER", "ubuntu"), ami: os.Getenv("DDIL_WITNESS_AMI"), instanceType: os.Getenv("DDIL_WITNESS_INSTANCE_TYPE"), keyName: envDefault("DDIL_WITNESS_KEY_NAME", "spinifex-key"), diff --git a/tests/e2e/iam/datapath_helpers_test.go b/tests/e2e/iam/datapath_helpers_test.go index 760f98895..1ba317a08 100644 --- a/tests/e2e/iam/datapath_helpers_test.go +++ b/tests/e2e/iam/datapath_helpers_test.go @@ -142,7 +142,7 @@ func trySSHReady(host string, port int, keyPath string, timeout time.Duration) b "-o", "BatchMode=yes", "-p", strconv.Itoa(port), "-i", keyPath, - "ec2-user@"+host, + "ubuntu@"+host, "true") if cmd.Run() == nil { return true diff --git a/tests/e2e/iam/imds_test.go b/tests/e2e/iam/imds_test.go index d6128cd7a..37a3f0ecc 100644 --- a/tests/e2e/iam/imds_test.go +++ b/tests/e2e/iam/imds_test.go @@ -42,12 +42,17 @@ const ( // localport, no netns, no in-guest route. metaURL = "http://169.254.169.254" - // imdsUserData is a no-op cloud-config carrying a unique marker so the - // /latest/user-data round-trip can assert the body the guest sees matches - // what RunInstances was given. A comment-only cloud-config is a valid empty - // config, so it doesn't perturb boot. - imdsUserData = "#cloud-config\n# imds-e2e-userdata-marker-7f3a91\n" - imdsUDMarker = "imds-e2e-userdata-marker-7f3a91" + // imdsUserData is a #cloud-config carrying a unique marker, used two ways: the + // /latest/user-data round-trip asserts the body the guest sees matches what + // RunInstances was given, and its runcmd writes imdsUDMarker to imdsUDDoneFile + // so the boot-from-IMDS guard can prove cloud-init actually RAN user-data from + // the Ec2 datasource, not just that the responder served it. + imdsUserData = "#cloud-config\n" + + "# imds-e2e-userdata-marker-7f3a91\n" + + "runcmd:\n" + + " - [ sh, -c, \"echo imds-e2e-userdata-marker-7f3a91 > /run/imds-e2e-userdata.done\" ]\n" + imdsUDMarker = "imds-e2e-userdata-marker-7f3a91" + imdsUDDoneFile = "/run/imds-e2e-userdata.done" // imdsProbeVPCCIDR / imdsProbeSubnetCIDR are shared by BOTH isolation VPCs. // Overlapping CIDRs across VPCs are legal (each VPC is its own routing @@ -221,6 +226,9 @@ func runIMDS(t *testing.T, fix *Fixture) { "no public IP → public-hostname must 404") } + // VM X is profile-bound, so meta-data/ lists iam/ and cloud-init descends. + require.Contains(t, imdsGet(t, tgtX, tokenX, "/latest/meta-data/"), "iam/", + "profile-bound meta-data/ listing must include iam/") // iam/info → InstanceProfileArn ends with the bound profile. iamInfo := imdsGet(t, tgtX, tokenX, "/latest/meta-data/iam/info") require.Containsf(t, iamInfo, ":instance-profile/"+imdsProfileName, @@ -251,6 +259,13 @@ func runIMDS(t *testing.T, fix *Fixture) { require.Contains(t, imdsGet(t, tgtX, tokenX, "/latest/user-data"), imdsUDMarker, "user-data must round-trip the launch user-data") + // --- Boot-from-IMDS cutover guards --------------------------------------- + // The rest of this suite proves the guest can READ IMDS; this proves it BOOTED + // from it. The NoCloud seed is retired, so every guest now self-configures from + // the Ec2 datasource — and every other assertion here still passes if the seed + // silently comes back, so this is the only guard against that regression. + imdsAssertBootFromIMDS(t, tgtX, privX) + // --- Version discovery + dated-version alias (cloud-init parity) ---------- // cloud-init's EC2 datasource probes its OWN hardcoded dated versions // (e.g. 2021-03-23), not the GET / listing, so any dated prefix must alias to @@ -337,6 +352,19 @@ func runIMDS(t *testing.T, fix *Fixture) { "/latest/meta-data/instance-id"), "a token bound to VM X's ENI must not authorise VM Y") + // VM Y has no instance profile, so the whole iam/ subtree is absent: real EC2 + // omits iam/ from the meta-data/ listing and 404s the iam/ directory, so + // cloud-init never descends and never trips on a 404ing iam/info that would fail + // its metadata crawl and zombie the guest. + harness.Step(t, "VM Y meta-data/ omits iam/; iam/ directory 404s") + require.NotContains(t, imdsGet(t, tgtY, tokenY, "/latest/meta-data/"), "iam/", + "no-profile meta-data/ listing must omit iam/") + for _, p := range []string{"/latest/meta-data/iam", "/latest/meta-data/iam/"} { + require.Equalf(t, "404", + imdsCode(tgtY, fmt.Sprintf(`-H "X-aws-ec2-metadata-token: %s"`, tokenY), p), + "no-profile %s must 404 (real-EC2 parity)", p) + } + // VM Y has no instance profile: iam/info is 404 and the credential listing is // an empty 200 (absence is not an error, matching AWS). harness.Step(t, "VM Y iam/info → 404; security-credentials/ → empty 200") @@ -391,7 +419,7 @@ func imdsProbe(t *testing.T, fix *Fixture, keyPath string, spec imdsVMSpec) (str host, port := harness.InstancePublicSSHHost(t, inst) harness.Step(t, "wait for %s SSH at %s:%d", id, host, port) waitForSSHHandshake(t, host, port, keyPath) - return id, priv, eni, harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + return id, priv, eni, harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} } // imdsAssertPerTapDatapath asserts the per-tap IMDS datapath is realised on the @@ -485,6 +513,60 @@ func imdsAssertVpcdFileCaps(t *testing.T) { } } +// imdsAssertBootFromIMDS proves VM X bootstrapped from the Ec2 IMDS datasource +// with the NoCloud seed retired — a regression guard the rest of the suite can't +// give: every other assertion still passes if the seed silently comes back and +// cloud-init boots from NoCloud instead. Checks, all in-guest: +// +// - cloud-init reports the aws platform / an Ec2 datasource (not NoCloud); +// - no cidata-labelled block device is attached (the seed ISO is gone); +// - the login is the AMI's stock default user (no Spinifex-forced account); +// - the hostname is the AWS form ip-; +// - the primary NIC carries the VPC IP rendered from IMDS; +// - the user-data runcmd executed (cloud-init processed user-data from IMDS). +func imdsAssertBootFromIMDS(t *testing.T, tgt harness.SSHTarget, privIP string) { + t.Helper() + harness.Step(t, "boot-from-IMDS: cloud-init selected the aws (Ec2) datasource") + ds := strings.ToLower(strings.TrimSpace( + runSSH(t, tgt, "cloud-id 2>/dev/null || cloud-init query --format '{{datasource}}'"))) + require.Truef(t, strings.Contains(ds, "aws") || strings.Contains(ds, "ec2"), + "cloud-init must report the aws/Ec2 datasource (got %q) — not the retired NoCloud seed", ds) + + harness.Step(t, "boot-from-IMDS: no NoCloud cidata seed device attached") + labels := strings.ToLower(runSSH(t, tgt, "lsblk -no LABEL")) + require.NotContainsf(t, labels, "cidata", + "no cidata-labelled device may be attached — the seed ISO is retired (lsblk LABELs: %q)", labels) + + harness.Step(t, "boot-from-IMDS: login is the AMI stock default user %q", tgt.User) + require.Equal(t, tgt.User, strings.TrimSpace(runSSH(t, tgt, "id -un")), + "in-guest login must be the AMI stock default user, not a Spinifex-forced account") + + harness.Step(t, "boot-from-IMDS: AWS-form hostname ip-") + wantHost := "ip-" + strings.ReplaceAll(privIP, ".", "-") + gotHost := strings.TrimSpace(runSSH(t, tgt, "hostname")) + require.Truef(t, strings.HasPrefix(gotHost, wantHost), + "hostname must be the AWS form %q rendered from IMDS local-hostname (got %q)", wantHost, gotHost) + + harness.Step(t, "boot-from-IMDS: primary NIC up with the VPC IP %s", privIP) + addrs := runSSH(t, tgt, "ip -4 -o addr show scope global") + require.Containsf(t, addrs, privIP, + "the Ec2 datasource must render the primary NIC with the VPC IP %s from IMDS (ip addr: %q)", privIP, addrs) + + harness.Step(t, "boot-from-IMDS: user-data runcmd executed") + deadline := time.Now().Add(90 * time.Second) + for { + out, _ := runSSHCombined(tgt, "cat "+imdsUDDoneFile+" 2>/dev/null") + if strings.Contains(out, imdsUDMarker) { + break + } + if time.Now().After(deadline) { + t.Fatalf("user-data runcmd marker %s never appeared within 90s — "+ + "cloud-init did not run user-data from the Ec2 datasource", imdsUDDoneFile) + } + time.Sleep(3 * time.Second) + } +} + // imdsShortENI mirrors host.shortENIID: the FNV-32a hash of the full ENI ID as 8 // hex chars, which the per-tap port names key off (inlined, as this suite inlines // the other OVS/OVN names). A hash, NOT a truncation — suffix-sharing ENIs differ. diff --git a/tests/e2e/multinode/baseline_test.go b/tests/e2e/multinode/baseline_test.go index 2495638fb..fe0a5d850 100644 --- a/tests/e2e/multinode/baseline_test.go +++ b/tests/e2e/multinode/baseline_test.go @@ -137,12 +137,12 @@ func runMultinodeDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { harness.Step(t, "authorizing tcp/22, expecting reachability") harness.AuthorizeSSHIngress(t, fix.AWS, sgID) - harness.GuestSSHReady(t, pubIP, 22, "ec2-user", keyPath, + harness.GuestSSHReady(t, pubIP, 22, "ubuntu", keyPath, harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second)) - out, err := sshCapture(keyPath, "ec2-user", pubIP, 22, "id") + out, err := sshCapture(keyPath, "ubuntu", pubIP, 22, "id") require.NoErrorf(t, err, "ssh id after authorize: %s", out) - assert.Containsf(t, out, "ec2-user", "ssh id after authorize\n%s", out) + assert.Containsf(t, out, "ubuntu", "ssh id after authorize\n%s", out) } // runMultinodeSameSGCrossHostComms launches two instances on different nodes sharing @@ -197,11 +197,11 @@ func runMultinodeSameSGCrossHostComms(t *testing.T, fix *Fixture) { // Shell into the source instance via its dedicated runner-SSH SG. host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, src.id) - harness.GuestSSHReady(t, host, port, "ec2-user", keyPath, + harness.GuestSSHReady(t, host, port, "ubuntu", keyPath, harness.WithTimeout(3*time.Minute), harness.WithPoll(3*time.Second)) harness.Step(t, "ping %s (%s) from %s across hosts via default-SG self-ingress", dst.id, dstPriv, src.id) - out, err := sshCapture(keyPath, "ec2-user", host, port, + out, err := sshCapture(keyPath, "ubuntu", host, port, fmt.Sprintf("ping -c 3 -W 2 %s", dstPriv)) require.NoErrorf(t, err, "cross-host ping %s -> %s failed; default SG self-ingress not enforced across chassis\n%s", diff --git a/tests/e2e/multinode/guest_ssh_test.go b/tests/e2e/multinode/guest_ssh_test.go index 4d44a91e1..8b704f78f 100644 --- a/tests/e2e/multinode/guest_ssh_test.go +++ b/tests/e2e/multinode/guest_ssh_test.go @@ -17,8 +17,9 @@ import ( // runGuestSSH is the Go port of guest SSH probes // (run-multinode-e2e.sh:626-728). For every instance in the trio, resolve an // SSH endpoint (PublicIpAddress or hosting-node hostfwd), wait until SSH -// answers, and assert `id` reports ec2-user. Also runs `lsblk` as a smoke -// probe for the root-device wiring. +// answers, and assert `id` reports ubuntu (the AMI's stock default user, now +// that the seed is retired and cloud-init bootstraps from the Ec2 datasource). +// Also asserts the AWS-form hostname and runs `lsblk` as a root-device smoke. func runGuestSSH(t *testing.T, fix *Fixture) { harness.Phase(t, "Multinode — Guest SSH") @@ -30,16 +31,25 @@ func runGuestSSH(t *testing.T, fix *Fixture) { host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, id) harness.Detail(t, "instance", id, "ssh_host", host, "ssh_port", port) - harness.GuestSSHReady(t, host, port, "ec2-user", pemPath, + harness.GuestSSHReady(t, host, port, "ubuntu", pemPath, harness.WithTimeout(2*time.Minute), harness.WithPoll(2*time.Second)) - idOut := sshRun(t, pemPath, "ec2-user", host, port, "id") + idOut := sshRun(t, pemPath, "ubuntu", host, port, "id") harness.Detail(t, "instance", id, "id", strings.TrimSpace(idOut)) - if !strings.Contains(idOut, "ec2-user") { - t.Fatalf("instance %s id output missing ec2-user:\n%s", id, idOut) + if !strings.Contains(idOut, "ubuntu") { + t.Fatalf("instance %s id output missing ubuntu:\n%s", id, idOut) } - lsblk := sshRun(t, pemPath, "ec2-user", host, port, "lsblk") + // Cutover guard: cloud-init sets the AWS-form hostname ip- from + // IMDS local-hostname, not the retired seed's spinifex-vm-. + hostname := strings.TrimSpace(sshRun(t, pemPath, "ubuntu", host, port, "hostname")) + harness.Detail(t, "instance", id, "hostname", hostname) + if !strings.HasPrefix(hostname, "ip-") { + t.Fatalf("instance %s hostname %q is not the AWS form ip- "+ + "rendered from IMDS — seed may not be retired", id, hostname) + } + + lsblk := sshRun(t, pemPath, "ubuntu", host, port, "lsblk") // lsblk header line + at least one device row. harness.Detail(t, "instance", id, "lsblk_lines", strings.Count(lsblk, "\n")) } diff --git a/tests/e2e/multinode/placement_nat_test.go b/tests/e2e/multinode/placement_nat_test.go index 262e2c9af..d2ce0fe42 100644 --- a/tests/e2e/multinode/placement_nat_test.go +++ b/tests/e2e/multinode/placement_nat_test.go @@ -632,7 +632,7 @@ func waitForBastionSSH(t *testing.T, host, keyPath string, timeout time.Duration "-o", "ConnectTimeout=5", "-o", "BatchMode=yes", "-i", keyPath, - "ec2-user@" + host, + "ubuntu@" + host, "true", } if out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput(); err != nil { @@ -658,7 +658,7 @@ func scpKeyToBastion(t *testing.T, keyPath, host string) { "-o", "BatchMode=yes", "-i", keyPath, keyPath, - "ec2-user@" + host + ":/tmp/key.pem", + "ubuntu@" + host + ":/tmp/key.pem", } if out, err := exec.CommandContext(ctx, "scp", scpArgs...).CombinedOutput(); err != nil { t.Fatalf("scp %s -> %s:/tmp/key.pem: %v\n%s", keyPath, host, err, string(out)) @@ -673,7 +673,7 @@ func scpKeyToBastion(t *testing.T, keyPath, host string) { "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", "-i", keyPath, - "ec2-user@" + host, + "ubuntu@" + host, "chmod 600 /tmp/key.pem", } if out, err := exec.CommandContext(ctx2, "ssh", sshArgs...).CombinedOutput(); err != nil { @@ -688,7 +688,7 @@ func sshViaBastion(keyPath, bastionHost, privIP, cmd string) (string, error) { hop := fmt.Sprintf( "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "+ "-o LogLevel=ERROR -o ConnectTimeout=10 -o BatchMode=yes "+ - "-i /tmp/key.pem ec2-user@%s '%s'", + "-i /tmp/key.pem ubuntu@%s '%s'", privIP, cmd, ) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) @@ -700,7 +700,7 @@ func sshViaBastion(keyPath, bastionHost, privIP, cmd string) (string, error) { "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", "-i", keyPath, - "ec2-user@" + bastionHost, + "ubuntu@" + bastionHost, hop, } out, err := exec.CommandContext(ctx, "ssh", args...).CombinedOutput() diff --git a/tests/e2e/multinode/volume_durability_test.go b/tests/e2e/multinode/volume_durability_test.go index 7aad5d8b3..968472d2c 100644 --- a/tests/e2e/multinode/volume_durability_test.go +++ b/tests/e2e/multinode/volume_durability_test.go @@ -101,7 +101,7 @@ func pickCrossNodePair(t *testing.T, fix *Fixture, trio []string) (srcID, dstID, func guestTarget(t *testing.T, fix *Fixture, instanceID, pemPath string) harness.SSHTarget { t.Helper() host, port := harness.GuestSSHEndpoint(t, fix.AWS, fix.Cluster, instanceID) - if !harness.TryGuestSSHReady(host, port, "ec2-user", pemPath, 2*time.Minute) { + if !harness.TryGuestSSHReady(host, port, "ubuntu", pemPath, 2*time.Minute) { privIP := harness.InstancePrivateIP(t, fix.AWS, instanceID) harness.DumpVPCFlowDiagnostics(t, fix.AWS, instanceID, fmt.Sprintf("volume-durability guest SSH timeout — instance=%s pub=%s priv=%s", @@ -114,5 +114,5 @@ func guestTarget(t *testing.T, fix *Fixture, instanceID, pemPath string) harness t.Fatalf("guest %s SSH %s:%d never became reachable within 2min (see diagnostics above)", instanceID, host, port) } - return harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: pemPath} + return harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: pemPath} } diff --git a/tests/e2e/reboot/reboot_test.go b/tests/e2e/reboot/reboot_test.go index 5db57e509..b948ae546 100644 --- a/tests/e2e/reboot/reboot_test.go +++ b/tests/e2e/reboot/reboot_test.go @@ -1339,7 +1339,7 @@ ls -l /var/lib/cloud/instances/*/sem/ 2>&1 umask 077 && printf '%%s' '%s' | base64 -d > %s && chmod 600 %s ssh -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ -o ConnectTimeout=8 -o BatchMode=yes -o ServerAliveInterval=5 \ - ec2-user@%s bash -s <<'EOFDIAG' + ubuntu@%s bash -s <<'EOFDIAG' %s EOFDIAG rm -f %s diff --git a/tests/e2e/run-bm-multinode-e2e.sh b/tests/e2e/run-bm-multinode-e2e.sh index c51fa531b..6148b83d2 100755 --- a/tests/e2e/run-bm-multinode-e2e.sh +++ b/tests/e2e/run-bm-multinode-e2e.sh @@ -731,7 +731,7 @@ for idx in "${!INSTANCE_IDS[@]}"; do -o LogLevel=ERROR \ -p "$SSH_PORT" \ -i "$SSH_KEY" \ - ec2-user@"$SSH_HOST" 'echo ready' > /dev/null 2>&1; then + ubuntu@"$SSH_HOST" 'echo ready' > /dev/null 2>&1; then SSH_READY=true break fi @@ -755,17 +755,17 @@ for idx in "${!INSTANCE_IDS[@]}"; do -o LogLevel=ERROR \ -p "$SSH_PORT" \ -i "$SSH_KEY" \ - ec2-user@"$SSH_HOST" 'id' 2>&1) || { + ubuntu@"$SSH_HOST" 'id' 2>&1) || { echo " ERROR: SSH 'id' command failed" fail_test "Guest SSH ($instance_id)" continue } echo " SSH 'id' output: $ID_OUTPUT" - if echo "$ID_OUTPUT" | grep -q "ec2-user"; then - echo " ec2-user confirmed" + if echo "$ID_OUTPUT" | grep -q "ubuntu"; then + echo " ubuntu confirmed" else - echo " ERROR: Expected 'ec2-user' in id output" + echo " ERROR: Expected 'ubuntu' in id output" fail_test "Guest SSH ($instance_id)" continue fi @@ -778,7 +778,7 @@ for idx in "${!INSTANCE_IDS[@]}"; do -o LogLevel=ERROR \ -p "$SSH_PORT" \ -i "$SSH_KEY" \ - ec2-user@"$SSH_HOST" 'lsblk' 2>&1) || true + ubuntu@"$SSH_HOST" 'lsblk' 2>&1) || true echo " lsblk: $(echo "$LSBLK_OUTPUT" | head -5)" pass_test "Guest SSH ($instance_id)" @@ -1490,7 +1490,7 @@ BASTION_OK=false for attempt in $(seq 1 30); do if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ -o LogLevel=ERROR -o ConnectTimeout=5 -o BatchMode=yes \ - -i "$HOME/.ssh/spinifex-key" "ec2-user@$NATGW_BASTION_PUB_IP" "true" 2>/dev/null; then + -i "$HOME/.ssh/spinifex-key" "ubuntu@$NATGW_BASTION_PUB_IP" "true" 2>/dev/null; then BASTION_OK=true break fi @@ -1515,19 +1515,19 @@ else # Copy SSH key to bastion for private instance hops scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ -i "$HOME/.ssh/spinifex-key" "$HOME/.ssh/spinifex-key" \ - "ec2-user@$NATGW_BASTION_PUB_IP:/tmp/key.pem" 2>/dev/null + "ubuntu@$NATGW_BASTION_PUB_IP:/tmp/key.pem" 2>/dev/null ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \ - -i "$HOME/.ssh/spinifex-key" "ec2-user@$NATGW_BASTION_PUB_IP" "chmod 600 /tmp/key.pem" 2>/dev/null + -i "$HOME/.ssh/spinifex-key" "ubuntu@$NATGW_BASTION_PUB_IP" "chmod 600 /tmp/key.pem" 2>/dev/null # Bastion SSH helper bastion_ssh() { ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=10 \ - -i "$HOME/.ssh/spinifex-key" "ec2-user@$NATGW_BASTION_PUB_IP" "$@" + -i "$HOME/.ssh/spinifex-key" "ubuntu@$NATGW_BASTION_PUB_IP" "$@" } priv_hop_cmd() { local priv_ip="$1"; shift - echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=10 -o BatchMode=yes -i /tmp/key.pem ec2-user@${priv_ip} '$*'" + echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=10 -o BatchMode=yes -i /tmp/key.pem ubuntu@${priv_ip} '$*'" } echo "" diff --git a/tests/e2e/run-tofu-examples-e2e.sh b/tests/e2e/run-tofu-examples-e2e.sh index 51a885893..bcaa09b34 100755 --- a/tests/e2e/run-tofu-examples-e2e.sh +++ b/tests/e2e/run-tofu-examples-e2e.sh @@ -110,7 +110,7 @@ trap cleanup EXIT wait_for_ssh() { local key="$1" host="$2" for _ in $(seq 1 30); do - if ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${host}" true 2>/dev/null; then + if ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" true 2>/dev/null; then return 0 fi sleep 5 @@ -163,17 +163,17 @@ assert_bastion_private_subnet() { } # sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem. # Wait up to 180s for user_data to drop the key before hopping. - if ! ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \ + if ! ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" \ 'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1'; then log " bastion: ~/.ssh/bastion-demo.pem never appeared (cloud-init stalled?)" return 1 fi - local inner_ssh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes -i ~/.ssh/bastion-demo.pem ec2-user@${private}" + local inner_ssh="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o BatchMode=yes -i ~/.ssh/bastion-demo.pem ubuntu@${private}" local attempt for attempt in $(seq 1 30); do - if ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" "${inner_ssh} true" 2>/dev/null; then - ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" "${inner_ssh} id" | grep -q '^uid=' + if ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} true" 2>/dev/null; then + ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} id" | grep -q '^uid=' return $? fi sleep 5 @@ -243,7 +243,7 @@ dump_s3_webapp_guest() { fi chmod 600 "$key" 2>/dev/null || true log "--- s3-webapp guest: cloud-init status + output log + unit status ---" - ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${host}" ' + ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" ' echo "== cloud-init status =="; cloud-init status --long 2>/dev/null || true echo "== /var/log/cloud-init-output.log (tail 80) ==" sudo tail -n 80 /var/log/cloud-init-output.log 2>/dev/null || true diff --git a/tests/e2e/single/baseline_test.go b/tests/e2e/single/baseline_test.go index 5094d2191..09eed5363 100644 --- a/tests/e2e/single/baseline_test.go +++ b/tests/e2e/single/baseline_test.go @@ -183,9 +183,9 @@ func runDefaultSGReachabilityBaseline(t *testing.T, fix *Fixture) { "tcp/22 to %s never became reachable after authorizing ingress — "+ "default subnet egress/IGW datapath is broken", pubIP) - tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath} idOut := runSSH(t, tgt, "id") - assert.Containsf(t, idOut, "ec2-user", "ssh id after authorize\n%s", idOut) + assert.Containsf(t, idOut, "ubuntu", "ssh id after authorize\n%s", idOut) } // mainRouteTableID returns the main (implicitly-associated) route table for @@ -311,9 +311,9 @@ func runNewVPCEgressBaseline(t *testing.T, fix *Fixture) { "after the main-RT IGW route existed", pubIP, subnetID) } - tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath} idOut := runSSH(t, tgt, "id") - assert.Containsf(t, idOut, "ec2-user", "ssh id in fresh-VPC subnet\n%s", idOut) + assert.Containsf(t, idOut, "ubuntu", "ssh id in fresh-VPC subnet\n%s", idOut) } // runSameSGComms verifies that two instances in the default SG can ICMP-ping @@ -341,7 +341,7 @@ func runSameSGComms(t *testing.T, fix *Fixture) { harness.Detail(t, "src", srcID, "dst", dstID, "dst_private_ip", dstPriv, "ssh_host", host) waitForSSHHandshake(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} harness.Step(t, "ping %s (%s) from %s via default-SG self-ingress", dstID, dstPriv, srcID) out, converged := pingConverged(tgt, dstPriv, 45*time.Second) require.Truef(t, converged, diff --git a/tests/e2e/single/datapath_test.go b/tests/e2e/single/datapath_test.go index 0c9932141..861d9fbfb 100644 --- a/tests/e2e/single/datapath_test.go +++ b/tests/e2e/single/datapath_test.go @@ -185,7 +185,7 @@ func runSGToSGDatapath(t *testing.T, fix *Fixture) { // `ssh 'curl :8080'`. Target-vm has no SSH ingress on // its own SG so we never connect to it directly. clientHost, clientPort := harness.InstancePublicSSHHost(t, clientInst) - clientTgt := harness.SSHTarget{User: "ec2-user", Host: clientHost, Port: clientPort, KeyPath: keyPath} + clientTgt := harness.SSHTarget{User: "ubuntu", Host: clientHost, Port: clientPort, KeyPath: keyPath} harness.Step(t, "8e-3 wait for client-vm SSH at %s:%d", clientHost, clientPort) // Non-fatal probe so a timeout dumps the guest console + OVN/datapath // state before Fatal. A full 2min unreachable window on a fresh public-IP diff --git a/tests/e2e/single/eip_test.go b/tests/e2e/single/eip_test.go index c48702261..62d463b58 100644 --- a/tests/e2e/single/eip_test.go +++ b/tests/e2e/single/eip_test.go @@ -117,9 +117,9 @@ func runInstanceEIP(t *testing.T, fix *Fixture) { }) t.Fatalf("guest unreachable via EIP %s within %s (see diagnostics above)", eipIP, sshReadyBudget) } - tgt := harness.SSHTarget{User: "ec2-user", Host: eipIP, Port: 22, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: eipIP, Port: 22, KeyPath: keyPath} idOut := runSSH(t, tgt, "id") - require.Containsf(t, idOut, "ec2-user", "ssh via EIP id did not report ec2-user\n%s", idOut) + require.Containsf(t, idOut, "ubuntu", "ssh via EIP id did not report ubuntu\n%s", idOut) harness.Detail(t, "datapath", "eip_reachable_ok") // --- DisassociateAddress + assert unreachable ------------------------- diff --git a/tests/e2e/single/eni_test.go b/tests/e2e/single/eni_test.go index 3c57c6e15..b21c87f46 100644 --- a/tests/e2e/single/eni_test.go +++ b/tests/e2e/single/eni_test.go @@ -41,7 +41,7 @@ func runENIHotplug(t *testing.T, fix *Fixture) { // completes; an unreachable singleton is a datapath bug, not test flake. host, port := harness.InstancePublicSSHHost(t, inst) waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} // Snapshot the guest's MAC inventory before the hotplug so we detect the // brand-new interface rather than match a NIC that already existed. diff --git a/tests/e2e/single/instance_test.go b/tests/e2e/single/instance_test.go index 750e900bc..0d3568a73 100644 --- a/tests/e2e/single/instance_test.go +++ b/tests/e2e/single/instance_test.go @@ -103,11 +103,11 @@ func runSSHProbe(t *testing.T, fix *Fixture) { waitForSSHHandshake(t, host, port, keyPath) _ = addr - tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} harness.Step(t, "ssh id") idOut := runSSH(t, tgt, "id") - assert.Containsf(t, idOut, "ec2-user", "ssh id should report ec2-user\n%s", idOut) + assert.Containsf(t, idOut, "ubuntu", "ssh id should report ubuntu\n%s", idOut) harness.Step(t, "lsblk root-volume cross-check vs API") guestGiB := harness.LsblkRootGiB(t, tgt) diff --git a/tests/e2e/single/lifecycle_test.go b/tests/e2e/single/lifecycle_test.go index c743a4e9f..d528db116 100644 --- a/tests/e2e/single/lifecycle_test.go +++ b/tests/e2e/single/lifecycle_test.go @@ -259,7 +259,7 @@ func runModifyInstanceAttribute(t *testing.T, fix *Fixture) { harness.Detail(t, "ssh_host", host, "ssh_port", port) waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} harness.Step(t, "ssh nproc") nprocOut := strings.TrimSpace(runSSH(t, tgt, "nproc")) @@ -341,7 +341,7 @@ func runRebootInstance(t *testing.T, fix *Fixture) { host, port = harness.InstancePublicSSHHost(t, postInst) waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} harness.Step(t, "ssh uptime") uptimeOut := strings.TrimSpace(runSSH(t, tgt, "cat /proc/uptime | cut -d. -f1")) uptimeSecs, err := strconv.ParseInt(uptimeOut, 10, 64) @@ -461,7 +461,7 @@ func trySSHReady(host string, port int, keyPath string, timeout time.Duration) b "-o", "BatchMode=yes", "-p", strconv.Itoa(port), "-i", keyPath, - "ec2-user@"+host, + "ubuntu@"+host, "true") if cmd.Run() == nil { return true diff --git a/tests/e2e/single/natgw_test.go b/tests/e2e/single/natgw_test.go index ba057e286..c26283faa 100644 --- a/tests/e2e/single/natgw_test.go +++ b/tests/e2e/single/natgw_test.go @@ -128,10 +128,10 @@ func runNATGateway(t *testing.T, fix *Fixture) { // Wait for bastion SSH handshake so the SCP below has somewhere to land. waitForSSHReady(t, bastionPubIP, 22, keyPath) - bastionTgt := harness.SSHTarget{User: "ec2-user", Host: bastionPubIP, Port: 22, KeyPath: keyPath} + bastionTgt := harness.SSHTarget{User: "ubuntu", Host: bastionPubIP, Port: 22, KeyPath: keyPath} // Copy the keypair to the bastion so it can hop into the private VM. - // Matches the bash `scp ... ec2-user@$PUB_IP:/tmp/key.pem` step. + // Matches the bash `scp ... ubuntu@$PUB_IP:/tmp/key.pem` step. harness.Step(t, "scp keypair -> bastion:/tmp/key.pem") scpKey(t, keyPath, bastionPubIP) _ = runSSH(t, bastionTgt, "chmod 600 /tmp/key.pem") @@ -141,7 +141,7 @@ func runNATGateway(t *testing.T, fix *Fixture) { privProbe := fmt.Sprintf( "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "+ "-o LogLevel=ERROR -o ConnectTimeout=10 -o BatchMode=yes "+ - "-i /tmp/key.pem ec2-user@%s hostname", + "-i /tmp/key.pem ubuntu@%s hostname", privIP, ) harness.Step(t, "wait for private SSH via bastion") @@ -161,7 +161,7 @@ func runNATGateway(t *testing.T, fix *Fixture) { return fmt.Sprintf( "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "+ "-o LogLevel=ERROR -o ConnectTimeout=10 -o BatchMode=yes "+ - "-i /tmp/key.pem ec2-user@%s 'ping -c 1 -W 3 8.8.8.8'", + "-i /tmp/key.pem ubuntu@%s 'ping -c 1 -W 3 8.8.8.8'", privIP, ) } @@ -412,7 +412,7 @@ func waitForInstanceStateSoft(c *harness.AWSClient, id, target string, timeout t } // scpKey copies the harness PEM to /tmp/key.pem on the bastion. Matches -// the bash `scp -i $KEY $KEY ec2-user@$PUB_IP:/tmp/key.pem` step — the +// the bash `scp -i $KEY $KEY ubuntu@$PUB_IP:/tmp/key.pem` step — the // private VM then accepts that key for the hop. func scpKey(t *testing.T, keyPath, host string) { t.Helper() @@ -424,7 +424,7 @@ func scpKey(t *testing.T, keyPath, host string) { "-o", "BatchMode=yes", "-i", keyPath, keyPath, - "ec2-user@" + host + ":/tmp/key.pem", + "ubuntu@" + host + ":/tmp/key.pem", } cmd := exec.Command("scp", args...) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/tests/e2e/single/securitygroup_test.go b/tests/e2e/single/securitygroup_test.go index fe36ab265..21e9418d7 100644 --- a/tests/e2e/single/securitygroup_test.go +++ b/tests/e2e/single/securitygroup_test.go @@ -40,7 +40,7 @@ func runSecurityGroupEgress(t *testing.T, fix *Fixture) { require.NotEmpty(t, def.SGID, "default SG ID required") sshHost, sshPort := harness.InstancePublicSSHHost(t, inst) - tgt := harness.SSHTarget{User: "ec2-user", Host: sshHost, Port: sshPort, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: sshHost, Port: sshPort, KeyPath: keyPath} // Restore allow-all egress no matter what — ignore Duplicate so a clean // finish (test left the rule in place) doesn't poison later phases. diff --git a/tests/e2e/single/volume_durability_test.go b/tests/e2e/single/volume_durability_test.go index 6008d3926..15b9a9452 100644 --- a/tests/e2e/single/volume_durability_test.go +++ b/tests/e2e/single/volume_durability_test.go @@ -40,7 +40,7 @@ func runVolumeDurability(t *testing.T, fix *Fixture) { host, port := harness.InstancePublicSSHHost(t, inst) waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} // 1. Create volume + assert the encryption-wiring signal. harness.Step(t, "create-volume size=1 az=%s", az) @@ -94,7 +94,7 @@ func runVolumeDurability(t *testing.T, fix *Fixture) { // SSH endpoint may have rebound after the restart — re-discover. host, port = harness.InstancePublicSSHHost(t, runInst) waitForSSHReady(t, host, port, keyPath) - tgt = harness.SSHTarget{User: "ec2-user", Host: host, Port: port, KeyPath: keyPath} + tgt = harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} gotSha = harness.GuestReadSentinelSha(t, tgt, "/dev/disk/by-label/"+volumeDataLabel, volumeDataLabel) require.Equalf(t, wantSha, gotSha, "sha256 mismatch after stop/start") diff --git a/tests/e2e/single/vpc_test.go b/tests/e2e/single/vpc_test.go index 269ca76ea..0c1777839 100644 --- a/tests/e2e/single/vpc_test.go +++ b/tests/e2e/single/vpc_test.go @@ -259,11 +259,11 @@ func runVPCSubnetE2E(t *testing.T, fix *Fixture) { t.Fatalf("SSH handshake %s:22 never completed within %s (see diagnostics above)", pubIP, sshReadyBudget) } - tgt := harness.SSHTarget{User: "ec2-user", Host: pubIP, Port: 22, KeyPath: keyPath} + tgt := harness.SSHTarget{User: "ubuntu", Host: pubIP, Port: 22, KeyPath: keyPath} harness.Step(t, "ssh id (smoke)") idOut := runSSH(t, tgt, "id") - require.Containsf(t, idOut, "ec2-user", "ssh id did not report ec2-user\n%s", idOut) + require.Containsf(t, idOut, "ubuntu", "ssh id did not report ubuntu\n%s", idOut) // Verify external connectivity through the IGW + 0.0.0.0/0 route. ping // is treated the same way bash treats it: a soft check that prints a