From 31d71e0390d6943905b16fbbbade679eb230b668 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Tue, 21 Jul 2026 08:46:25 +1000 Subject: [PATCH] test(e2e): merge single-node guest churn tests into one durability scenario Combines TestSSHProbe, TestENIHotplug, TestENIHotplugReconcile, TestVolumeDurability, TestModifyInstanceAttribute, and TestRebootInstance into TestGuestChurnDurability, a single shared-guest scenario that re-verifies a data-integrity sentinel sha256 after every perturbation stage (ENI hotplug, daemon restart, ENI detach, instance-type resize, stop/start, reboot). This proves the sentinel survives the full churn sequence rather than just one isolated stop/start. --- tests/e2e/manifest-lint/baseline.txt | 1 - tests/e2e/single/eni_reconcile_test.go | 193 ----- tests/e2e/single/eni_test.go | 262 ------- tests/e2e/single/guestchurn_test.go | 780 +++++++++++++++++++++ tests/e2e/single/instance_test.go | 64 -- tests/e2e/single/lifecycle_test.go | 202 +----- tests/e2e/single/tests_test.go | 38 +- tests/e2e/single/volume_durability_test.go | 100 --- 8 files changed, 789 insertions(+), 851 deletions(-) delete mode 100644 tests/e2e/single/eni_reconcile_test.go delete mode 100644 tests/e2e/single/eni_test.go create mode 100644 tests/e2e/single/guestchurn_test.go delete mode 100644 tests/e2e/single/volume_durability_test.go diff --git a/tests/e2e/manifest-lint/baseline.txt b/tests/e2e/manifest-lint/baseline.txt index 172d2d997..c88467050 100644 --- a/tests/e2e/manifest-lint/baseline.txt +++ b/tests/e2e/manifest-lint/baseline.txt @@ -79,7 +79,6 @@ create tests/e2e/single/negative_test.go RunInstances #2 create tests/e2e/single/negative_test.go RunInstances #3 create tests/e2e/single/negative_test.go RunInstances #4 create tests/e2e/single/snapshot_test.go CreateSnapshot #1 -create tests/e2e/single/volume_durability_test.go CreateVolume #1 create tests/e2e/single/volume_test.go CreateVolume #1 create tests/e2e/single/vpc_test.go CreateInternetGateway #1 create tests/e2e/single/vpc_test.go CreateRouteTable #1 diff --git a/tests/e2e/single/eni_reconcile_test.go b/tests/e2e/single/eni_reconcile_test.go deleted file mode 100644 index 1f32de564..000000000 --- a/tests/e2e/single/eni_reconcile_test.go +++ /dev/null @@ -1,193 +0,0 @@ -//go:build e2e - -package single - -import ( - "fmt" - "os/exec" - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/mulgadc/spinifex/tests/e2e/harness" - "github.com/stretchr/testify/require" -) - -// runENIHotplugReconcile proves the Sprint 3d hot-plug reconciler end to end: -// attach a secondary ENI, restart spinifex-daemon (the guest QEMU survives via -// KillMode=process), then assert the ENI stays attached and is still fully -// manageable — DescribeNetworkInterfaces reports in-use, the guest NIC is still -// present, and a detach succeeds. The detach is the load-bearing assertion: the -// hot-attach mutates the in-memory PCIe slot map, which may not have been -// persisted before the restart, so without the reconciler re-adopting the slot -// the detach would fail InvalidAttachmentID.NotFound. OVN-gated; restores the -// singleton (detach + delete) before returning. -func runENIHotplugReconcile(t *testing.T, fix *Fixture) { - harness.Phase(t, "Single — ENI Hot-Plug Reconciler Convergence (OVN)") - harness.SkipIfNoOVN(t) - requireSSHHealthy(t) - - c := fix.AWS - inst, _ := needInstance(t, fix) - instanceID := aws.StringValue(inst.InstanceId) - _, keyPath := needKeyPair(t, fix) - - def := harness.EnsureDefaultVPC(t, fix.Harness) - require.NotEmpty(t, def.SubnetID, "default subnet ID required") - require.NotEmpty(t, def.SGID, "default SG ID required") - harness.AuthorizeSSHIngress(t, c, def.SGID) - - host, port := harness.InstancePublicSSHHost(t, inst) - waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} - - before := guestMACSet(t, tgt) - - // --- Attach a secondary ENI ------------------------------------------- - harness.Step(t, "create-network-interface subnet=%s", def.SubnetID) - // e2e:allow-create — the secondary ENI is the subject under test. - cniOut, err := c.EC2.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ - SubnetId: aws.String(def.SubnetID), - Groups: []*string{aws.String(def.SGID)}, - Description: aws.String("e2e-eni-reconcile"), - }) - require.NoError(t, err, "create-network-interface") - require.NotNil(t, cniOut.NetworkInterface, "create-network-interface returned nil ENI") - eniID := aws.StringValue(cniOut.NetworkInterface.NetworkInterfaceId) - eniMAC := strings.ToLower(aws.StringValue(cniOut.NetworkInterface.MacAddress)) - require.NotEmpty(t, eniID, "ENI NetworkInterfaceId empty") - require.NotEmpty(t, eniMAC, "ENI MacAddress empty") - harness.Detail(t, "eni", eniID, "eni_mac", eniMAC) - - t.Cleanup(func() { - if _, derr := c.EC2.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ - NetworkInterfaceId: aws.String(eniID), - }); derr != nil && !harness.ErrorCodeIs(derr, "InvalidNetworkInterfaceID.NotFound") { - t.Logf("WARNING: cleanup delete ENI %s: %v", eniID, derr) - } - }) - - harness.Step(t, "attach-network-interface %s -> %s device-index=1", eniID, instanceID) - attachOut, err := c.EC2.AttachNetworkInterface(&ec2.AttachNetworkInterfaceInput{ - NetworkInterfaceId: aws.String(eniID), - InstanceId: aws.String(instanceID), - DeviceIndex: aws.Int64(1), - }) - require.NoError(t, err, "attach-network-interface") - attachmentID := aws.StringValue(attachOut.AttachmentId) - require.NotEmpty(t, attachmentID, "AttachmentId empty") - detached := false - t.Cleanup(func() { - if detached { - return - } - if _, derr := c.EC2.DetachNetworkInterface(&ec2.DetachNetworkInterfaceInput{ - AttachmentId: aws.String(attachmentID), - }); derr != nil && !harness.ErrorCodeIs(derr, "InvalidAttachmentID.NotFound") { - t.Logf("WARNING: cleanup detach ENI %s: %v", eniID, derr) - } - _ = waitForENIStatusSoft(c, eniID, "available", 2*time.Minute) - }) - - waitForENIAttached(t, c, eniID, instanceID) - require.NotContains(t, before, eniMAC, "ENI MAC present before attach — cannot prove a fresh hotplug") - harness.Step(t, "assert guest NIC with MAC %s appears", eniMAC) - harness.EventuallyErr(t, func() error { - if _, ok := guestMACSet(t, tgt)[eniMAC]; ok { - return nil - } - return fmt.Errorf("ENI MAC %s not yet visible in guest", eniMAC) - }, 90*time.Second, 3*time.Second) - - // --- Restart the daemon; the guest must survive ------------------------ - harness.Step(t, "restart spinifex-daemon (guest QEMU survives)") - restartSpinifexDaemon(t) - waitForControlPlaneReady(t, c, instanceID) - - // --- Post-restart: ENI still attached + guest NIC intact -------------- - harness.Step(t, "assert ENI %s still attached after restart", eniID) - waitForENIAttached(t, c, eniID, instanceID) - require.Contains(t, guestMACSet(t, tgt), eniMAC, - "guest NIC for MAC %s vanished across daemon restart — QEMU should survive", eniMAC) - - // --- Detach: succeeds only if the reconciler re-adopted the slot ------ - harness.Step(t, "detach-network-interface %s (proves slot re-adoption)", attachmentID) - detachAfterReconcile(t, c, attachmentID) - waitForENIStatus(t, c, eniID, "available") - detached = true - - harness.Step(t, "verify guest NIC for MAC %s disappears", eniMAC) - if !guestNICDropped(t, tgt, eniMAC, 45*time.Second) { - t.Logf("WARN: guest NIC for MAC %s still present after detach (udev lag?)", eniMAC) - } - - harness.Step(t, "delete-network-interface %s", eniID) - _, err = c.EC2.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ - NetworkInterfaceId: aws.String(eniID), - }) - require.NoError(t, err, "delete-network-interface") -} - -// restartSpinifexDaemon restarts the host spinifex-daemon unit. The single-node -// suite runs on the host with passwordless sudo (same precedent as harness diag -// capture), so a local systemctl restart reaches the daemon without SSH. -func restartSpinifexDaemon(t *testing.T) { - t.Helper() - out, err := exec.Command("sudo", "-n", "systemctl", "restart", "spinifex-daemon").CombinedOutput() - require.NoErrorf(t, err, "restart spinifex-daemon: %s", strings.TrimSpace(string(out))) -} - -// waitForControlPlaneReady polls DescribeInstances until the daemon has -// reconnected after a restart and serves the singleton again. -func waitForControlPlaneReady(t *testing.T, c *harness.AWSClient, instanceID string) { - t.Helper() - harness.EventuallyErr(t, func() error { - out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - if err != nil { - return fmt.Errorf("describe-instances after restart: %w", err) - } - if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 { - return fmt.Errorf("instance %s not yet visible after restart", instanceID) - } - return nil - }, 90*time.Second, 3*time.Second) -} - -// detachAfterReconcile issues the detach, retrying while the reconciler's -// startup sweep is still re-adopting the slot. InvalidAttachmentID.NotFound is -// the symptom of an un-adopted slot and is safe to retry (no state changed); -// any other error is fatal. -func detachAfterReconcile(t *testing.T, c *harness.AWSClient, attachmentID string) { - t.Helper() - harness.EventuallyErr(t, func() error { - _, err := c.EC2.DetachNetworkInterface(&ec2.DetachNetworkInterfaceInput{ - AttachmentId: aws.String(attachmentID), - }) - if err == nil { - return nil - } - if harness.ErrorCodeIs(err, "InvalidAttachmentID.NotFound") { - return fmt.Errorf("slot not yet re-adopted by reconciler: %w", err) - } - t.Fatalf("detach-network-interface %s: %v", attachmentID, err) - return nil - }, 60*time.Second, 3*time.Second) -} - -// guestNICDropped polls briefly for the guest NIC carrying mac to disappear. -// Soft: virtio-net unplug + udev cleanup can lag the control-plane detach. -func guestNICDropped(t *testing.T, tgt harness.SSHTarget, mac string, timeout time.Duration) bool { - t.Helper() - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - if _, ok := guestMACSet(t, tgt)[mac]; !ok { - return true - } - time.Sleep(3 * time.Second) - } - return false -} diff --git a/tests/e2e/single/eni_test.go b/tests/e2e/single/eni_test.go deleted file mode 100644 index b21c87f46..000000000 --- a/tests/e2e/single/eni_test.go +++ /dev/null @@ -1,262 +0,0 @@ -//go:build e2e - -package single - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/mulgadc/spinifex/tests/e2e/harness" - "github.com/stretchr/testify/require" -) - -// runENIHotplug exercises the secondary-ENI hotplug datapath end to end: -// CreateNetworkInterface in the singleton's subnet, AttachNetworkInterface at -// DeviceIndex 1 to the running guest, then assert the new virtio-net NIC -// surfaces inside the guest kernel carrying the ENI's MAC. Proves the QMP -// device_add path (gateway -> ec2.cmd. daemon -> QEMU hot-plug) reaches -// the guest, not just the control plane. OVN-gated; reuses the singleton VM -// and restores it (detach + delete) before returning. -func runENIHotplug(t *testing.T, fix *Fixture) { - harness.Phase(t, "Single — Secondary ENI Hotplug Datapath (OVN)") - harness.SkipIfNoOVN(t) - requireSSHHealthy(t) - - c := fix.AWS - inst, _ := needInstance(t, fix) - instanceID := aws.StringValue(inst.InstanceId) - _, keyPath := needKeyPair(t, fix) - - def := harness.EnsureDefaultVPC(t, fix.Harness) - require.NotEmpty(t, def.SubnetID, "default subnet ID required") - require.NotEmpty(t, def.SGID, "default SG ID required") - harness.AuthorizeSSHIngress(t, c, def.SGID) - - // SSH to the singleton via its public IP — the in-guest `ip link` snapshot - // needs a live datapath. Fatal (with sticky-skip) if the handshake never - // completes; an unreachable singleton is a datapath bug, not test flake. - host, port := harness.InstancePublicSSHHost(t, inst) - waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} - - // Snapshot the guest's MAC inventory before the hotplug so we detect the - // brand-new interface rather than match a NIC that already existed. - before := guestMACSet(t, tgt) - harness.Detail(t, "guest_macs_before", strings.Join(setKeys(before), ",")) - - // --- CreateNetworkInterface ------------------------------------------- - harness.Step(t, "create-network-interface subnet=%s", def.SubnetID) - // e2e:allow-create — the secondary ENI is the subject hot-plugged under test. - cniOut, err := c.EC2.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ - SubnetId: aws.String(def.SubnetID), - Groups: []*string{aws.String(def.SGID)}, - Description: aws.String("e2e-eni-hotplug"), - }) - require.NoError(t, err, "create-network-interface") - require.NotNil(t, cniOut.NetworkInterface, "create-network-interface returned nil ENI") - eniID := aws.StringValue(cniOut.NetworkInterface.NetworkInterfaceId) - eniMAC := strings.ToLower(aws.StringValue(cniOut.NetworkInterface.MacAddress)) - eniIP := aws.StringValue(cniOut.NetworkInterface.PrivateIpAddress) - require.NotEmpty(t, eniID, "ENI NetworkInterfaceId empty") - require.NotEmpty(t, eniMAC, "ENI MacAddress empty") - require.NotEmpty(t, eniIP, "ENI PrivateIpAddress empty") - harness.Detail(t, "eni", eniID, "eni_mac", eniMAC, "eni_ip", eniIP) - - // Delete cleanup runs last (LIFO); the detach cleanup registered next must - // flip the ENI back to "available" before this fires. - t.Cleanup(func() { - if _, derr := c.EC2.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ - NetworkInterfaceId: aws.String(eniID), - }); derr != nil && !harness.ErrorCodeIs(derr, "InvalidNetworkInterfaceID.NotFound") { - t.Logf("WARNING: cleanup delete ENI %s: %v", eniID, derr) - } - }) - - // --- AttachNetworkInterface (hot-plug) -------------------------------- - harness.Step(t, "attach-network-interface %s -> %s device-index=1", eniID, instanceID) - attachOut, err := c.EC2.AttachNetworkInterface(&ec2.AttachNetworkInterfaceInput{ - NetworkInterfaceId: aws.String(eniID), - InstanceId: aws.String(instanceID), - DeviceIndex: aws.Int64(1), - }) - require.NoError(t, err, "attach-network-interface") - attachmentID := aws.StringValue(attachOut.AttachmentId) - require.NotEmpty(t, attachmentID, "AttachmentId empty") - detached := false - t.Cleanup(func() { - if detached { - return - } - if _, derr := c.EC2.DetachNetworkInterface(&ec2.DetachNetworkInterfaceInput{ - AttachmentId: aws.String(attachmentID), - }); derr != nil && !harness.ErrorCodeIs(derr, "InvalidAttachmentID.NotFound") { - t.Logf("WARNING: cleanup detach ENI %s: %v", eniID, derr) - } - _ = waitForENIStatusSoft(c, eniID, "available", 2*time.Minute) - }) - harness.Detail(t, "attachment", attachmentID) - - // --- Control plane: ENI shows attached to the singleton --------------- - harness.Step(t, "wait ENI %s in-use attached to %s", eniID, instanceID) - waitForENIAttached(t, c, eniID, instanceID) - - // --- Datapath: new NIC surfaces in the guest carrying the ENI MAC ----- - harness.Step(t, "assert guest NIC with MAC %s appears", eniMAC) - if _, existed := before[eniMAC]; existed { - t.Fatalf("ENI MAC %s was already present before attach — cannot prove a fresh hotplug", eniMAC) - } - harness.EventuallyErr(t, func() error { - now := guestMACSet(t, tgt) - if _, ok := now[eniMAC]; ok { - return nil - } - return fmt.Errorf("ENI MAC %s not yet visible in guest (have: %s)", - eniMAC, strings.Join(setKeys(now), ",")) - }, 90*time.Second, 3*time.Second) - harness.Detail(t, "datapath", "guest_nic_hotplugged_ok") - - // Cross-check the control-plane secondary IP is the one assigned to the - // hotplugged ENI. In-guest IP autoconfiguration of the secondary NIC is - // OS/cloud-init dependent, so the IP assertion stays at the API level. - eni, err := describeENI(c, eniID) - require.NoError(t, err, "describe ENI %s after attach", eniID) - require.Equal(t, eniIP, aws.StringValue(eni.PrivateIpAddress), - "attached ENI %s private IP drifted from create-time value", eniID) - - // --- Detach + assert the NIC drops out of the guest ------------------- - harness.Step(t, "detach-network-interface %s", attachmentID) - _, err = c.EC2.DetachNetworkInterface(&ec2.DetachNetworkInterfaceInput{ - AttachmentId: aws.String(attachmentID), - }) - require.NoError(t, err, "detach-network-interface") - waitForENIStatus(t, c, eniID, "available") - detached = true - - // Guest NIC removal is a soft check: virtio-net unplug + udev cleanup can - // lag the control-plane detach, and a lingering link doesn't break the EC2 - // contract the way a missing hotplug would. Poll briefly, warn if it stays. - harness.Step(t, "verify guest NIC for MAC %s disappears", eniMAC) - dropped := false - deadline := time.Now().Add(45 * time.Second) - for time.Now().Before(deadline) { - if _, ok := guestMACSet(t, tgt)[eniMAC]; !ok { - dropped = true - break - } - time.Sleep(3 * time.Second) - } - if dropped { - harness.Detail(t, "datapath", "guest_nic_removed_ok") - } else { - t.Logf("WARN: guest NIC for MAC %s still present after detach (udev lag?)", eniMAC) - } - - // --- DeleteNetworkInterface ------------------------------------------- - harness.Step(t, "delete-network-interface %s", eniID) - _, err = c.EC2.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ - NetworkInterfaceId: aws.String(eniID), - }) - require.NoError(t, err, "delete-network-interface") -} - -// guestMACSet returns the set of lowercased MAC addresses currently present on -// the guest's network interfaces, read from sysfs so the list tracks the live -// kernel view (including hotplugged NICs). The all-zero loopback MAC is -// dropped so it never masks a real match. -func guestMACSet(t *testing.T, tgt harness.SSHTarget) map[string]struct{} { - t.Helper() - out, err := harness.GuestExec(tgt, "cat /sys/class/net/*/address") - if err != nil { - t.Fatalf("guest read MAC inventory: %v\n%s", err, out) - } - set := map[string]struct{}{} - for _, line := range strings.Split(strings.TrimSpace(out), "\n") { - mac := strings.ToLower(strings.TrimSpace(line)) - if mac == "" || mac == "00:00:00:00:00:00" { - continue - } - set[mac] = struct{}{} - } - return set -} - -// setKeys returns the keys of a string set as a slice for log/Detail output. -func setKeys(m map[string]struct{}) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - return keys -} - -// describeENI returns the single ENI record for eniID, erroring if absent. -func describeENI(c *harness.AWSClient, eniID string) (*ec2.NetworkInterface, error) { - out, err := c.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ - NetworkInterfaceIds: []*string{aws.String(eniID)}, - }) - if err != nil { - return nil, fmt.Errorf("describe-network-interfaces %s: %w", eniID, err) - } - if len(out.NetworkInterfaces) == 0 { - return nil, fmt.Errorf("ENI %s not found", eniID) - } - return out.NetworkInterfaces[0], nil -} - -// waitForENIAttached polls until eniID reports Status=in-use with an -// Attachment bound to instanceID. -func waitForENIAttached(t *testing.T, c *harness.AWSClient, eniID, instanceID string) { - t.Helper() - harness.EventuallyErr(t, func() error { - eni, err := describeENI(c, eniID) - if err != nil { - return err - } - status := aws.StringValue(eni.Status) - if eni.Attachment == nil { - return fmt.Errorf("ENI %s status=%s has no attachment yet", eniID, status) - } - if got := aws.StringValue(eni.Attachment.InstanceId); got != instanceID { - return fmt.Errorf("ENI %s attached to %q want %q", eniID, got, instanceID) - } - if status != "in-use" { - return fmt.Errorf("ENI %s status=%s want in-use", eniID, status) - } - return nil - }, 90*time.Second, 3*time.Second) -} - -// waitForENIStatus polls until eniID reaches target status (e.g. "available"). -func waitForENIStatus(t *testing.T, c *harness.AWSClient, eniID, target string) { - t.Helper() - harness.EventuallyErr(t, func() error { - eni, err := describeENI(c, eniID) - if err != nil { - return err - } - if got := aws.StringValue(eni.Status); got != target { - return fmt.Errorf("ENI %s status=%s want=%s", eniID, got, target) - } - return nil - }, 90*time.Second, 3*time.Second) -} - -// waitForENIStatusSoft is the cleanup-time variant of waitForENIStatus: never -// calls t.Fatal, just polls best-effort within timeout. -func waitForENIStatusSoft(c *harness.AWSClient, eniID, target string, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for { - eni, err := describeENI(c, eniID) - if err == nil && aws.StringValue(eni.Status) == target { - return nil - } - if time.Now().After(deadline) { - return fmt.Errorf("ENI %s did not reach %s within %s", eniID, target, timeout) - } - time.Sleep(2 * time.Second) - } -} diff --git a/tests/e2e/single/guestchurn_test.go b/tests/e2e/single/guestchurn_test.go new file mode 100644 index 000000000..ef1da0def --- /dev/null +++ b/tests/e2e/single/guestchurn_test.go @@ -0,0 +1,780 @@ +//go:build e2e + +package single + +import ( + "fmt" + "net" + "os/exec" + "strconv" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// guestChurnSentinelLabel / guestChurnSentinelSizeMiB parameterise the +// durability sentinel written once in WriteSentinel and re-checked after +// every later perturbation. A 1 GiB volume is the AWS minimum; the 4 MiB +// random payload is large enough to be a meaningful checksum target while +// keeping the predastore round-trip fast. +const ( + guestChurnSentinelLabel = "e2edur" + guestChurnSentinelSizeMiB = 4 +) + +// guestChurnRounds returns how many times the full perturbation sequence +// (HotplugENI through Reboot) repeats against the same guest and sentinel, +// overridable via SPINIFEX_GUESTCHURN_ROUNDS. Defaults to 1, matching every +// former standalone test's single-pass behaviour; bump it for a soak run +// that repeatedly churns the guest to catch reconciler or durability +// flakiness a single pass would not surface. +func guestChurnRounds() int { + return envPositiveIntOr("SPINIFEX_GUESTCHURN_ROUNDS", 1) +} + +// runGuestChurnDurability merges six checks that used to boot their own +// singleton apiece — SSH/root-volume probe, ENI hotplug, the hot-plug +// reconciler, volume data durability, ModifyInstanceAttribute, and reboot — +// around one shared guest and one shared data-integrity sentinel. +// +// This merge is not just a time-saver: today VolumeDurability alone proves a +// sha256 survives one detach/reattach and one stop/start. Merged, the same +// sha is re-verified after ENI hotplug, a daemon restart, an instance-type +// change, a fresh stop/start, and a reboot — proving the write path holds up +// across every kind of churn this suite exercises, not just one of them. +// +// Stage order and gating: +// - BootAndRootVolume boots (or adopts) the singleton and proves basic +// SSH and root-volume sanity. Nothing below it can run against a guest +// that never booted, so its failure is fatal for the whole scenario +// (require in the ordinary Go-test sense, via a hard t.Fatalf gate). +// - WriteSentinel creates the data volume, writes the sentinel, and proves +// it survives one detach/reattach before any perturbation begins. Its +// failure is likewise fatal: every later re-verification compares +// against the sha captured here, so there is nothing to re-verify if +// this stage never captured a good one. +// - Every stage from HotplugENI onward is a perturbation of the running +// guest; a re-verification of the sentinel sha immediately follows each +// one, using require — a corrupted sentinel invalidates every later +// perturbation's result, so continuing past a bad readback would only +// produce misleading noise. +// - HotplugENI, DaemonRestartReadoptsSlot, and DetachENI form one ENI +// lifecycle: hotplug, prove the reconciler re-adopts the slot across a +// daemon restart, then detach. They share one ENI rather than each +// creating its own, which is what lets DaemonRestartReadoptsSlot prove +// reconciliation against an ENI that was already fully verified live, +// and what lets the final detach double as both "a working ENI detaches +// cleanly" and "detach succeeds only if the reconciler re-adopted the +// slot". OVN-gated (see below): if HotplugENI does not attach an ENI — +// whether skipped or failed — DaemonRestartReadoptsSlot and DetachENI +// do not run, since they have nothing to act on. This does NOT gate +// ResizeInstanceType, SurvivesStopStart, or Reboot: those perturb the +// guest and its data volume, not the secondary ENI, and must still run +// (and still get their own sentinel re-verification) in an environment +// with no OVN. +// - ResizeInstanceType's own t.Cleanup restores the original instance +// type and running state at the end of that stage specifically — not +// deferred to the end of the round — so SurvivesStopStart and Reboot +// always start from a clean, running, original-type instance regardless +// of whether ResizeInstanceType itself passed. +// - SurvivesStopStart's own body already re-reads and compares the +// sentinel sha as its defining assertion, so no separate +// re-verification call follows it — that would just be the same guest +// exec repeated for nothing new. +// - Reboot never touched the data volume in its original standalone form; +// the sentinel re-verification after it is new coverage this merge +// adds, not a preserved assertion. +// +// The whole sequence from HotplugENI through Reboot repeats for +// guestChurnRounds() rounds (default 1); each round gets its own ENI and its +// own instance-type round-trip so a soak run exercises fresh resource +// lifecycles rather than replaying stale IDs. +func runGuestChurnDurability(t *testing.T, fix *Fixture) { + harness.Phase(t, "Single — Guest Churn Durability: sentinel survives ENI hotplug, daemon restart, type change, reboot, and stop/start") + + // scenarioT anchors cleanup that must outlive a single t.Run block — the + // sentinel volume, in particular, must survive every stage and every + // round below, not just the stage that created it. + scenarioT := t + + az := needAZ(t, fix) + origType, _ := needInstanceTypeArch(t, fix) + _, keyPath := needKeyPair(t, fix) + + var instanceID string + + bootOK := t.Run("BootAndRootVolume", func(t *testing.T) { + inst, rootVolumeID := needInstance(t, fix) + instanceID = aws.StringValue(inst.InstanceId) + + host, port := harness.InstancePublicSSHHost(t, inst) + harness.Detail(t, "ssh_host", host, "ssh_port", port) + + addr := net.JoinHostPort(host, strconv.Itoa(port)) + harness.Step(t, "waiting for SSH handshake %s", addr) + waitForSSHHandshake(t, host, port, keyPath) + _ = addr + + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} + + harness.Step(t, "ssh id") + idOut := runSSH(t, tgt, "id") + assert.Containsf(t, idOut, "ubuntu", "ssh id should report ubuntu\n%s", idOut) + + harness.Step(t, "lsblk root-volume cross-check vs API") + guestGiB := harness.LsblkRootGiB(t, tgt) + + vols, err := fix.AWS.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{ + VolumeIds: []*string{aws.String(rootVolumeID)}, + }) + require.NoError(t, err, "describe-volumes %s", rootVolumeID) + require.NotEmpty(t, vols.Volumes, "no volume for %s", rootVolumeID) + apiGiB := int(aws.Int64Value(vols.Volumes[0].Size)) + + // lsblk rounds down — bash treats equality as required, but on backing + // stores where the VM's view is a hair under the API size the rounding + // loses 1 GiB. Allow ±1 to match the bash intent without flaking on it. + diff := guestGiB - apiGiB + if diff < 0 { + diff = -diff + } + assert.LessOrEqualf(t, diff, 1, + "root volume size mismatch: guest=%d GiB api=%d GiB", guestGiB, apiGiB) + harness.Detail(t, "guest_gib", guestGiB, "api_gib", apiGiB) + + harness.Step(t, "ssh hostname") + hn := strings.TrimSpace(runSSH(t, tgt, "hostname")) + // Bash uses `spinifex-vm-` and treats + // a missing prefix as a non-fatal warning. Replicate the soft check via + // t.Logf rather than asserting — the spx hostname format is not part of + // the EC2 surface contract. + shortID := strings.TrimPrefix(instanceID, "i-") + if len(shortID) > 8 { + shortID = shortID[:8] + } + if strings.Contains(hn, shortID) { + harness.Detail(t, "hostname", hn, "matches_short_id", shortID) + } else { + t.Logf("hostname %q does not contain short id %q (non-fatal)", hn, shortID) + } + }) + if !bootOK { + t.Fatalf("BootAndRootVolume stage failed; skipping every later stage — the guest never booted") + } + + var wantSha string + writeOK := t.Run("WriteSentinel", func(t *testing.T) { + tgt := resolveGuestChurnTarget(t, fix, instanceID, keyPath) + + harness.Step(t, "create-volume size=1 az=%s", az) + // e2e:allow-create — the sentinel volume is the subject under test. + createOut, err := fix.AWS.EC2.CreateVolume(&ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(az), + Size: aws.Int64(1), + }) + require.NoError(t, err, "create-volume") + volID := aws.StringValue(createOut.VolumeId) + require.NotEmpty(t, volID, "CreateVolume returned empty VolumeId") + harness.Detail(t, "volume", volID, "encrypted", aws.BoolValue(createOut.Encrypted)) + assert.Truef(t, aws.BoolValue(createOut.Encrypted), + "CreateVolume.Encrypted should be true (on-by-default cluster master key); "+ + "false means the control-plane→viperblockd key wiring is broken") + // Registered on scenarioT, not this stage's own t: the volume must + // survive every stage and round below, and only get torn down when + // the whole merged scenario ends. + harness.RegisterVolumeTeardown(scenarioT, fix.AWS, volID) + harness.WaitForVolumeState(t, fix.AWS, volID, "available", harness.WithPoll(500*time.Millisecond)) + + before := harness.GuestDiskSet(t, tgt) + harness.AttachVolumeWait(t, fix.AWS, volID, instanceID, "/dev/sdf") + dev := harness.WaitForNewGuestDisk(t, tgt, before, 60*time.Second) + harness.Detail(t, "guest_dev", dev) + wantSha = harness.GuestFormatWriteSentinel(t, tgt, dev, guestChurnSentinelLabel, guestChurnSentinelSizeMiB) + harness.Detail(t, "sha256", wantSha) + + // Detach → reattach → re-read by device; sha must survive. Proves the + // write itself round-tripped through the backend before any + // perturbation below begins. + harness.DetachVolumeWait(t, fix.AWS, volID) + before = harness.GuestDiskSet(t, tgt) + harness.AttachVolumeWait(t, fix.AWS, volID, instanceID, "/dev/sdf") + dev = harness.WaitForNewGuestDisk(t, tgt, before, 60*time.Second) + gotSha := harness.GuestReadSentinelSha(t, tgt, "/dev/"+dev, guestChurnSentinelLabel) + require.Equalf(t, wantSha, gotSha, "sha256 mismatch after detach/reattach") + harness.Detail(t, "reattach_sha_ok", gotSha) + }) + if !writeOK { + t.Fatalf("WriteSentinel stage failed; skipping every perturbation stage — there is no sentinel to re-verify") + } + + // verifySentinel re-reads the sentinel by ext4 label (robust to device + // renumbering across the perturbations below) and requires it match the + // sha captured in WriteSentinel. Always require: a corrupted sentinel + // invalidates every later perturbation's result. + verifySentinel := func(t *testing.T, stage string) { + t.Helper() + tgt := resolveGuestChurnTarget(t, fix, instanceID, keyPath) + gotSha := harness.GuestReadSentinelSha(t, tgt, "/dev/disk/by-label/"+guestChurnSentinelLabel, guestChurnSentinelLabel) + require.Equalf(t, wantSha, gotSha, "sentinel sha256 mismatch after %s", stage) + harness.Detail(t, "sentinel_verified_after", stage, "sha256", gotSha) + } + + rounds := guestChurnRounds() + for round := 1; round <= rounds; round++ { + t.Run(fmt.Sprintf("Round%d", round), func(t *testing.T) { + runGuestChurnRound(t, fix, instanceID, origType, keyPath, wantSha, verifySentinel) + }) + } +} + +// runGuestChurnRound drives one pass of HotplugENI through Reboot against +// the already-booted guest and already-written sentinel. +func runGuestChurnRound(t *testing.T, fix *Fixture, instanceID, origType, keyPath, wantSha string, verifySentinel func(t *testing.T, stage string)) { + // roundT anchors ENI cleanup: the ENI created in HotplugENI must survive + // DaemonRestartReadoptsSlot and DetachENI, i.e. outlive its own creating + // subtest, so its cleanup is registered on the round's own t rather than + // the HotplugENI subtest's t. + roundT := t + c := fix.AWS + + var eniID, eniMAC, attachmentID string + eniAttached := false + detachedByStage := false + + t.Run("HotplugENI", func(t *testing.T) { + harness.SkipIfNoOVN(t) + requireSSHHealthy(t) + + tgt := resolveGuestChurnTarget(t, fix, instanceID, keyPath) + def := harness.EnsureDefaultVPC(t, fix.Harness) + require.NotEmpty(t, def.SubnetID, "default subnet ID required") + require.NotEmpty(t, def.SGID, "default SG ID required") + harness.AuthorizeSSHIngress(t, c, def.SGID) + + // Snapshot the guest's MAC inventory before the hotplug so we detect + // the brand-new interface rather than match a NIC that already existed. + before := guestMACSet(t, tgt) + harness.Detail(t, "guest_macs_before", strings.Join(setKeys(before), ",")) + + harness.Step(t, "create-network-interface subnet=%s", def.SubnetID) + // e2e:allow-create — the secondary ENI is the subject hot-plugged under test. + cniOut, err := c.EC2.CreateNetworkInterface(&ec2.CreateNetworkInterfaceInput{ + SubnetId: aws.String(def.SubnetID), + Groups: []*string{aws.String(def.SGID)}, + Description: aws.String("e2e-eni-hotplug"), + }) + require.NoError(t, err, "create-network-interface") + require.NotNil(t, cniOut.NetworkInterface, "create-network-interface returned nil ENI") + eniID = aws.StringValue(cniOut.NetworkInterface.NetworkInterfaceId) + eniMAC = strings.ToLower(aws.StringValue(cniOut.NetworkInterface.MacAddress)) + eniIP := aws.StringValue(cniOut.NetworkInterface.PrivateIpAddress) + require.NotEmpty(t, eniID, "ENI NetworkInterfaceId empty") + require.NotEmpty(t, eniMAC, "ENI MacAddress empty") + require.NotEmpty(t, eniIP, "ENI PrivateIpAddress empty") + harness.Detail(t, "eni", eniID, "eni_mac", eniMAC, "eni_ip", eniIP) + + // Delete cleanup runs last (LIFO); the detach cleanup registered next + // must flip the ENI back to "available" before this fires. + roundT.Cleanup(func() { + if _, derr := c.EC2.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(eniID), + }); derr != nil && !harness.ErrorCodeIs(derr, "InvalidNetworkInterfaceID.NotFound") { + t.Logf("WARNING: cleanup delete ENI %s: %v", eniID, derr) + } + }) + + harness.Step(t, "attach-network-interface %s -> %s device-index=1", eniID, instanceID) + attachOut, err := c.EC2.AttachNetworkInterface(&ec2.AttachNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(eniID), + InstanceId: aws.String(instanceID), + DeviceIndex: aws.Int64(1), + }) + require.NoError(t, err, "attach-network-interface") + attachmentID = aws.StringValue(attachOut.AttachmentId) + require.NotEmpty(t, attachmentID, "AttachmentId empty") + roundT.Cleanup(func() { + if detachedByStage { + return + } + if _, derr := c.EC2.DetachNetworkInterface(&ec2.DetachNetworkInterfaceInput{ + AttachmentId: aws.String(attachmentID), + }); derr != nil && !harness.ErrorCodeIs(derr, "InvalidAttachmentID.NotFound") { + t.Logf("WARNING: cleanup detach ENI %s: %v", eniID, derr) + } + _ = waitForENIStatusSoft(c, eniID, "available", 2*time.Minute) + }) + harness.Detail(t, "attachment", attachmentID) + + harness.Step(t, "wait ENI %s in-use attached to %s", eniID, instanceID) + waitForENIAttached(t, c, eniID, instanceID) + + harness.Step(t, "assert guest NIC with MAC %s appears", eniMAC) + if _, existed := before[eniMAC]; existed { + t.Fatalf("ENI MAC %s was already present before attach — cannot prove a fresh hotplug", eniMAC) + } + harness.EventuallyErr(t, func() error { + now := guestMACSet(t, tgt) + if _, ok := now[eniMAC]; ok { + return nil + } + return fmt.Errorf("ENI MAC %s not yet visible in guest (have: %s)", + eniMAC, strings.Join(setKeys(now), ",")) + }, 90*time.Second, 3*time.Second) + harness.Detail(t, "datapath", "guest_nic_hotplugged_ok") + + // Cross-check the control-plane secondary IP is the one assigned to + // the hotplugged ENI. In-guest IP autoconfiguration of the secondary + // NIC is OS/cloud-init dependent, so the IP assertion stays at the + // API level. + eni, err := describeENI(c, eniID) + require.NoError(t, err, "describe ENI %s after attach", eniID) + require.Equal(t, eniIP, aws.StringValue(eni.PrivateIpAddress), + "attached ENI %s private IP drifted from create-time value", eniID) + + eniAttached = true + }) + verifySentinel(t, "HotplugENI") + + if eniAttached { + t.Run("DaemonRestartReadoptsSlot", func(t *testing.T) { + tgt := resolveGuestChurnTarget(t, fix, instanceID, keyPath) + + // Restart the daemon; the guest must survive. + harness.Step(t, "restart spinifex-daemon (guest QEMU survives)") + restartSpinifexDaemon(t) + waitForControlPlaneReady(t, c, instanceID) + + // Post-restart: ENI still attached + guest NIC intact. + harness.Step(t, "assert ENI %s still attached after restart", eniID) + waitForENIAttached(t, c, eniID, instanceID) + require.Contains(t, guestMACSet(t, tgt), eniMAC, + "guest NIC for MAC %s vanished across daemon restart — QEMU should survive", eniMAC) + }) + verifySentinel(t, "DaemonRestartReadoptsSlot") + + t.Run("DetachENI", func(t *testing.T) { + tgt := resolveGuestChurnTarget(t, fix, instanceID, keyPath) + + // Detach: succeeds only if the reconciler re-adopted the slot. + harness.Step(t, "detach-network-interface %s (proves slot re-adoption)", attachmentID) + detachAfterReconcile(t, c, attachmentID) + waitForENIStatus(t, c, eniID, "available") + detachedByStage = true + + // Guest NIC removal is a soft check: virtio-net unplug + udev + // cleanup can lag the control-plane detach. + harness.Step(t, "verify guest NIC for MAC %s disappears", eniMAC) + if !guestNICDropped(t, tgt, eniMAC, 45*time.Second) { + t.Logf("WARN: guest NIC for MAC %s still present after detach (udev lag?)", eniMAC) + } + + harness.Step(t, "delete-network-interface %s", eniID) + _, err := c.EC2.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(eniID), + }) + require.NoError(t, err, "delete-network-interface") + }) + verifySentinel(t, "DetachENI") + } else { + t.Logf("no ENI attached this round (HotplugENI skipped or failed); skipping DaemonRestartReadoptsSlot and DetachENI") + } + + t.Run("ResizeInstanceType", func(t *testing.T) { + // Bash strips the ".nano" suffix and appends ".small" — same family, + // more RAM at matching vCPU. Avoids xlarge (16 GiB) which the CI host + // can't satisfy. + if !strings.HasSuffix(origType, ".nano") { + t.Fatalf("phase7b: expected discovered instance type to end with .nano, got %q", origType) + } + modifyType := strings.TrimSuffix(origType, ".nano") + ".small" + harness.Detail(t, "from_type", origType, "to_type", modifyType) + + // Stop the instance first — ModifyInstanceAttribute on a running + // instance is rejected. Restore (type + state) at the end of this + // stage so SurvivesStopStart/Reboot see the canonical running + // original-type instance regardless of whether this stage passed. + harness.Step(t, "stop-instances %s (precondition for modify)", instanceID) + _, err := fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "stop-instances") + harness.WaitForInstanceState(t, fix.AWS, instanceID, "stopped") + t.Cleanup(func() { + _, _ = fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + harness.WaitForInstanceState(t, fix.AWS, instanceID, "stopped") + _, _ = fix.AWS.EC2.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{ + InstanceId: aws.String(instanceID), + InstanceType: &ec2.AttributeValue{Value: aws.String(origType)}, + }) + _, _ = fix.AWS.EC2.StartInstances(&ec2.StartInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") + }) + + // Look up expected vCPUs / memory for the upsized type so we can + // assert SSH-reported values match what the AWS surface advertises. + typesOut, err := fix.AWS.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}) + require.NoError(t, err, "describe-instance-types") + var expectedVCPUs int64 + var expectedMemMiB int64 + for _, it := range typesOut.InstanceTypes { + if aws.StringValue(it.InstanceType) != modifyType { + continue + } + if it.VCpuInfo != nil { + expectedVCPUs = aws.Int64Value(it.VCpuInfo.DefaultVCpus) + } + if it.MemoryInfo != nil { + expectedMemMiB = aws.Int64Value(it.MemoryInfo.SizeInMiB) + } + break + } + require.NotZero(t, expectedVCPUs, "%s missing VCpuInfo.DefaultVCpus", modifyType) + require.NotZero(t, expectedMemMiB, "%s missing MemoryInfo.SizeInMiB", modifyType) + harness.Detail(t, "expected_vcpus", expectedVCPUs, "expected_mem_mib", expectedMemMiB) + + harness.Step(t, "modify-instance-attribute %s type=%s", instanceID, modifyType) + _, err = fix.AWS.EC2.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{ + InstanceId: aws.String(instanceID), + InstanceType: &ec2.AttributeValue{Value: aws.String(modifyType)}, + }) + require.NoError(t, err, "modify-instance-attribute") + + // Verify describe-instances reflects the new type before we start. + descOut, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "describe-instances") + require.NotEmpty(t, descOut.Reservations, "no reservations for %s", instanceID) + require.NotEmpty(t, descOut.Reservations[0].Instances, "no instances for %s", instanceID) + gotType := aws.StringValue(descOut.Reservations[0].Instances[0].InstanceType) + require.Equalf(t, modifyType, gotType, + "ModifyInstanceAttribute did not stick: want %s got %s", modifyType, gotType) + + harness.Step(t, "start-instances %s", instanceID) + _, err = fix.AWS.EC2.StartInstances(&ec2.StartInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "start-instances") + + runInst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") + + // Re-discover SSH endpoint — qemu hostfwd may have rebound. + host, port := harness.InstancePublicSSHHost(t, runInst) + harness.Detail(t, "ssh_host", host, "ssh_port", port) + + waitForSSHReady(t, host, port, keyPath) + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} + + harness.Step(t, "ssh nproc") + nprocOut := strings.TrimSpace(runSSH(t, tgt, "nproc")) + vmVCPUs, err := strconv.ParseInt(nprocOut, 10, 64) + require.NoErrorf(t, err, "parse nproc output %q", nprocOut) + require.Equalf(t, expectedVCPUs, vmVCPUs, + "vCPU mismatch after modify: VM=%d expected=%d", vmVCPUs, expectedVCPUs) + harness.Detail(t, "vm_vcpus", vmVCPUs) + + harness.Step(t, "ssh MemTotal") + memOut := strings.TrimSpace(runSSH(t, tgt, "awk '/MemTotal/ {print $2}' /proc/meminfo")) + vmMemKB, err := strconv.ParseInt(memOut, 10, 64) + require.NoErrorf(t, err, "parse MemTotal output %q", memOut) + vmMemMiB := vmMemKB / 1024 + // 15% margin for kernel-reserved memory, matching the bash threshold. + expectedMemLow := expectedMemMiB * 85 / 100 + require.GreaterOrEqualf(t, vmMemMiB, expectedMemLow, + "memory too low after modify: VM=%d MiB expected>=%d MiB (target %d MiB)", + vmMemMiB, expectedMemLow, expectedMemMiB) + harness.Detail(t, "vm_mem_mib", vmMemMiB, "threshold_mib", expectedMemLow) + + // Cleanup restores original type + running state, registered above. + }) + verifySentinel(t, "ResizeInstanceType") + + t.Run("SurvivesStopStart", func(t *testing.T) { + // Stop/start (forces re-mount after instance restart) → re-read by + // label, since the device name can shift across the restart. This + // re-read IS the stage's defining assertion, so no separate + // verifySentinel call follows it. + harness.Step(t, "stop-instances %s", instanceID) + _, err := fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "stop-instances") + harness.WaitForInstanceState(t, fix.AWS, instanceID, "stopped") + + harness.Step(t, "start-instances %s", instanceID) + _, err = fix.AWS.EC2.StartInstances(&ec2.StartInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "start-instances") + runInst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") + + host, port := harness.InstancePublicSSHHost(t, runInst) + waitForSSHReady(t, host, port, keyPath) + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} + gotSha := harness.GuestReadSentinelSha(t, tgt, "/dev/disk/by-label/"+guestChurnSentinelLabel, guestChurnSentinelLabel) + require.Equalf(t, wantSha, gotSha, "sha256 mismatch after stop/start") + harness.Detail(t, "stopstart_sha_ok", gotSha) + }) + + t.Run("Reboot", func(t *testing.T) { + inst := describeGuestChurnInstance(t, fix, instanceID) + host, port := harness.InstancePublicSSHHost(t, inst) + waitForSSHReady(t, host, port, keyPath) + + // Capture pre-reboot private IP for the post-reboot identity check. + preDesc, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "describe-instances (pre-reboot)") + require.NotEmpty(t, preDesc.Reservations, "no reservations for %s", instanceID) + require.NotEmpty(t, preDesc.Reservations[0].Instances, "no instances for %s", instanceID) + preRebootIP := aws.StringValue(preDesc.Reservations[0].Instances[0].PrivateIpAddress) + harness.Detail(t, "pre_reboot_private_ip", preRebootIP) + + harness.Step(t, "reboot-instances %s", instanceID) + _, err = fix.AWS.EC2.RebootInstances(&ec2.RebootInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "reboot-instances") + + // Bash polls 10× at 1s checking the state stays "running" — EC2's + // reboot semantics don't transition the instance state at all. + for i := 0; i < 10; i++ { + out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "describe-instances during reboot poll %d", i) + require.NotEmpty(t, out.Reservations[0].Instances, "instance disappeared during reboot") + state := aws.StringValue(out.Reservations[0].Instances[0].State.Name) + require.Equalf(t, "running", state, + "instance unexpectedly left running state during reboot: %s", state) + time.Sleep(1 * time.Second) + } + + // SSH endpoint may have rebound after the guest restart (qemu hostfwd + // can shift), so re-discover. + descPost, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "describe-instances (post-reboot)") + require.NotEmpty(t, descPost.Reservations[0].Instances, "no instances post-reboot") + postInst := descPost.Reservations[0].Instances[0] + host, port = harness.InstancePublicSSHHost(t, postInst) + waitForSSHReady(t, host, port, keyPath) + + tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} + harness.Step(t, "ssh uptime") + uptimeOut := strings.TrimSpace(runSSH(t, tgt, "cat /proc/uptime | cut -d. -f1")) + uptimeSecs, err := strconv.ParseInt(uptimeOut, 10, 64) + require.NoErrorf(t, err, "parse uptime output %q", uptimeOut) + require.LessOrEqualf(t, uptimeSecs, int64(120), + "guest uptime %ds is > 120s — reboot may not have occurred", uptimeSecs) + harness.Detail(t, "uptime_secs", uptimeSecs) + + postRebootIP := aws.StringValue(postInst.PrivateIpAddress) + assert.Equalf(t, preRebootIP, postRebootIP, + "PrivateIpAddress changed across reboot: %s -> %s", preRebootIP, postRebootIP) + + // Leave instance running — later stages/rounds expect the singleton + // to still be up. + }) + verifySentinel(t, "Reboot") +} + +// describeGuestChurnInstance returns the current *ec2.Instance for +// instanceID. Every perturbation this scenario drives can change the +// instance's public hostfwd endpoint (type change, reboot, stop/start), so +// stages re-describe rather than trust an *ec2.Instance captured earlier. +func describeGuestChurnInstance(t *testing.T, fix *Fixture, instanceID string) *ec2.Instance { + t.Helper() + out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + require.NoError(t, err, "describe-instances %s", instanceID) + require.NotEmpty(t, out.Reservations, "no reservations for %s", instanceID) + require.NotEmpty(t, out.Reservations[0].Instances, "no instances for %s", instanceID) + return out.Reservations[0].Instances[0] +} + +// resolveGuestChurnTarget re-describes instanceID and returns a fresh SSH +// target, waiting for the handshake to succeed. The public hostfwd port can +// rebind across every perturbation this scenario drives, so every stage +// re-discovers it rather than trusting a target captured before the +// perturbation. +func resolveGuestChurnTarget(t *testing.T, fix *Fixture, instanceID, keyPath string) harness.SSHTarget { + t.Helper() + inst := describeGuestChurnInstance(t, fix, instanceID) + host, port := harness.InstancePublicSSHHost(t, inst) + waitForSSHReady(t, host, port, keyPath) + return harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} +} + +// guestMACSet returns the set of lowercased MAC addresses currently present on +// the guest's network interfaces, read from sysfs so the list tracks the live +// kernel view (including hotplugged NICs). The all-zero loopback MAC is +// dropped so it never masks a real match. +func guestMACSet(t *testing.T, tgt harness.SSHTarget) map[string]struct{} { + t.Helper() + out, err := harness.GuestExec(tgt, "cat /sys/class/net/*/address") + if err != nil { + t.Fatalf("guest read MAC inventory: %v\n%s", err, out) + } + set := map[string]struct{}{} + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + mac := strings.ToLower(strings.TrimSpace(line)) + if mac == "" || mac == "00:00:00:00:00:00" { + continue + } + set[mac] = struct{}{} + } + return set +} + +// setKeys returns the keys of a string set as a slice for log/Detail output. +func setKeys(m map[string]struct{}) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} + +// describeENI returns the single ENI record for eniID, erroring if absent. +func describeENI(c *harness.AWSClient, eniID string) (*ec2.NetworkInterface, error) { + out, err := c.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{ + NetworkInterfaceIds: []*string{aws.String(eniID)}, + }) + if err != nil { + return nil, fmt.Errorf("describe-network-interfaces %s: %w", eniID, err) + } + if len(out.NetworkInterfaces) == 0 { + return nil, fmt.Errorf("ENI %s not found", eniID) + } + return out.NetworkInterfaces[0], nil +} + +// waitForENIAttached polls until eniID reports Status=in-use with an +// Attachment bound to instanceID. +func waitForENIAttached(t *testing.T, c *harness.AWSClient, eniID, instanceID string) { + t.Helper() + harness.EventuallyErr(t, func() error { + eni, err := describeENI(c, eniID) + if err != nil { + return err + } + status := aws.StringValue(eni.Status) + if eni.Attachment == nil { + return fmt.Errorf("ENI %s status=%s has no attachment yet", eniID, status) + } + if got := aws.StringValue(eni.Attachment.InstanceId); got != instanceID { + return fmt.Errorf("ENI %s attached to %q want %q", eniID, got, instanceID) + } + if status != "in-use" { + return fmt.Errorf("ENI %s status=%s want in-use", eniID, status) + } + return nil + }, 90*time.Second, 3*time.Second) +} + +// waitForENIStatus polls until eniID reaches target status (e.g. "available"). +func waitForENIStatus(t *testing.T, c *harness.AWSClient, eniID, target string) { + t.Helper() + harness.EventuallyErr(t, func() error { + eni, err := describeENI(c, eniID) + if err != nil { + return err + } + if got := aws.StringValue(eni.Status); got != target { + return fmt.Errorf("ENI %s status=%s want=%s", eniID, got, target) + } + return nil + }, 90*time.Second, 3*time.Second) +} + +// waitForENIStatusSoft is the cleanup-time variant of waitForENIStatus: never +// calls t.Fatal, just polls best-effort within timeout. +func waitForENIStatusSoft(c *harness.AWSClient, eniID, target string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + eni, err := describeENI(c, eniID) + if err == nil && aws.StringValue(eni.Status) == target { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("ENI %s did not reach %s within %s", eniID, target, timeout) + } + time.Sleep(2 * time.Second) + } +} + +// restartSpinifexDaemon restarts the host spinifex-daemon unit. The single-node +// suite runs on the host with passwordless sudo (same precedent as harness diag +// capture), so a local systemctl restart reaches the daemon without SSH. +func restartSpinifexDaemon(t *testing.T) { + t.Helper() + out, err := exec.Command("sudo", "-n", "systemctl", "restart", "spinifex-daemon").CombinedOutput() + require.NoErrorf(t, err, "restart spinifex-daemon: %s", strings.TrimSpace(string(out))) +} + +// waitForControlPlaneReady polls DescribeInstances until the daemon has +// reconnected after a restart and serves the singleton again. +func waitForControlPlaneReady(t *testing.T, c *harness.AWSClient, instanceID string) { + t.Helper() + harness.EventuallyErr(t, func() error { + out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instanceID)}, + }) + if err != nil { + return fmt.Errorf("describe-instances after restart: %w", err) + } + if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 { + return fmt.Errorf("instance %s not yet visible after restart", instanceID) + } + return nil + }, 90*time.Second, 3*time.Second) +} + +// detachAfterReconcile issues the detach, retrying while the reconciler's +// startup sweep is still re-adopting the slot. InvalidAttachmentID.NotFound is +// the symptom of an un-adopted slot and is safe to retry (no state changed); +// any other error is fatal. +func detachAfterReconcile(t *testing.T, c *harness.AWSClient, attachmentID string) { + t.Helper() + harness.EventuallyErr(t, func() error { + _, err := c.EC2.DetachNetworkInterface(&ec2.DetachNetworkInterfaceInput{ + AttachmentId: aws.String(attachmentID), + }) + if err == nil { + return nil + } + if harness.ErrorCodeIs(err, "InvalidAttachmentID.NotFound") { + return fmt.Errorf("slot not yet re-adopted by reconciler: %w", err) + } + t.Fatalf("detach-network-interface %s: %v", attachmentID, err) + return nil + }, 60*time.Second, 3*time.Second) +} + +// guestNICDropped polls briefly for the guest NIC carrying mac to disappear. +// Soft: virtio-net unplug + udev cleanup can lag the control-plane detach. +func guestNICDropped(t *testing.T, tgt harness.SSHTarget, mac string, timeout time.Duration) bool { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if _, ok := guestMACSet(t, tgt)[mac]; !ok { + return true + } + time.Sleep(3 * time.Second) + } + return false +} diff --git a/tests/e2e/single/instance_test.go b/tests/e2e/single/instance_test.go index fe262da11..1859fc8ae 100644 --- a/tests/e2e/single/instance_test.go +++ b/tests/e2e/single/instance_test.go @@ -4,10 +4,8 @@ package single import ( "bytes" - "net" "os/exec" "strconv" - "strings" "testing" "github.com/aws/aws-sdk-go/aws" @@ -67,68 +65,6 @@ func runInstanceMetadata(t *testing.T, fix *Fixture) { ) } -// runSSHProbe waits for SSH to become reachable, then runs `id`, -// `lsblk`, and `hostname` against the guest to confirm the VM booted and -// presents the expected root volume. Maps to run-e2e.sh ~381–463. -func runSSHProbe(t *testing.T, fix *Fixture) { - harness.Phase(t, "Single — SSH Connectivity Phase 5a-ii — SSH Connectivity & Volume Verification Volume Verification") - - inst, rootVolumeID := needInstance(t, fix) - _, keyPath := needKeyPair(t, fix) - instanceID := aws.StringValue(inst.InstanceId) - - host, port := harness.InstancePublicSSHHost(t, inst) - harness.Detail(t, "ssh_host", host, "ssh_port", port) - - addr := net.JoinHostPort(host, strconv.Itoa(port)) - harness.Step(t, "waiting for SSH handshake %s", addr) - waitForSSHHandshake(t, host, port, keyPath) - _ = addr - - tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} - - harness.Step(t, "ssh id") - idOut := runSSH(t, tgt, "id") - assert.Containsf(t, idOut, "ubuntu", "ssh id should report ubuntu\n%s", idOut) - - harness.Step(t, "lsblk root-volume cross-check vs API") - guestGiB := harness.LsblkRootGiB(t, tgt) - - vols, err := fix.AWS.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{ - VolumeIds: []*string{aws.String(rootVolumeID)}, - }) - require.NoError(t, err, "describe-volumes %s", rootVolumeID) - require.NotEmpty(t, vols.Volumes, "no volume for %s", rootVolumeID) - apiGiB := int(aws.Int64Value(vols.Volumes[0].Size)) - - // lsblk rounds down — bash treats equality as required, but on backing - // stores where the VM's view is a hair under the API size the rounding - // loses 1 GiB. Allow ±1 to match the bash intent without flaking on it. - diff := guestGiB - apiGiB - if diff < 0 { - diff = -diff - } - assert.LessOrEqualf(t, diff, 1, - "root volume size mismatch: guest=%d GiB api=%d GiB", guestGiB, apiGiB) - harness.Detail(t, "guest_gib", guestGiB, "api_gib", apiGiB) - - harness.Step(t, "ssh hostname") - hn := strings.TrimSpace(runSSH(t, tgt, "hostname")) - // Bash uses `spinifex-vm-` and treats - // a missing prefix as a non-fatal warning. Replicate the soft check via - // t.Logf rather than asserting — the spx hostname format is not part of - // the EC2 surface contract. - shortID := strings.TrimPrefix(instanceID, "i-") - if len(shortID) > 8 { - shortID = shortID[:8] - } - if strings.Contains(hn, shortID) { - harness.Detail(t, "hostname", hn, "matches_short_id", shortID) - } else { - t.Logf("hostname %q does not contain short id %q (non-fatal)", hn, shortID) - } -} - // runConsoleOutput verifies GetConsoleOutput round-trips the // instance ID. The payload itself is base64 cloud-init output; bash doesn't // decode it and neither do we. Maps to run-e2e.sh ~465–478. diff --git a/tests/e2e/single/lifecycle_test.go b/tests/e2e/single/lifecycle_test.go index 88efe6b46..8292ba4d8 100644 --- a/tests/e2e/single/lifecycle_test.go +++ b/tests/e2e/single/lifecycle_test.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "strconv" - "strings" "sync/atomic" "testing" "time" @@ -15,7 +14,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/tests/e2e/harness" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -160,204 +158,6 @@ func runAttachToStoppedError(t *testing.T, fix *Fixture) { cleaned = true } -// runModifyInstanceAttribute changes the (stopped) instance type to -// the same-family ".small" upsize, starts the instance, SSHes in to -// verify the new vCPU + memory budget, then stops the instance again so -// Phase 7c-pre can drive its own start/reboot cycle. Maps to run-e2e.sh -// ~1070–1177. -func runModifyInstanceAttribute(t *testing.T, fix *Fixture) { - harness.Phase(t, "Single — ModifyInstanceAttribute") - - inst, _ := needInstance(t, fix) - instanceID := aws.StringValue(inst.InstanceId) - origType, _ := needInstanceTypeArch(t, fix) - _, keyPath := needKeyPair(t, fix) - - // Bash strips the ".nano" suffix and appends ".small" — same family, - // more RAM at matching vCPU. Avoids xlarge (16 GiB) which the CI host - // can't satisfy. - if !strings.HasSuffix(origType, ".nano") { - t.Fatalf("phase7b: expected discovered instance type to end with .nano, got %q", origType) - } - modifyType := strings.TrimSuffix(origType, ".nano") + ".small" - harness.Detail(t, "from_type", origType, "to_type", modifyType) - - // Stop the instance first — ModifyInstanceAttribute on a running - // instance is rejected. Restore (type + state) at end so sibling Test* - // see the canonical running singleton. - harness.Step(t, "stop-instances %s (precondition for modify)", instanceID) - _, err := fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "stop-instances") - harness.WaitForInstanceState(t, fix.AWS, instanceID, "stopped") - t.Cleanup(func() { - _, _ = fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - harness.WaitForInstanceState(t, fix.AWS, instanceID, "stopped") - _, _ = fix.AWS.EC2.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{ - InstanceId: aws.String(instanceID), - InstanceType: &ec2.AttributeValue{Value: aws.String(origType)}, - }) - _, _ = fix.AWS.EC2.StartInstances(&ec2.StartInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") - }) - - // Look up expected vCPUs / memory for the upsized type so we can - // assert SSH-reported values match what the AWS surface advertises. - typesOut, err := fix.AWS.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{}) - require.NoError(t, err, "describe-instance-types") - var expectedVCPUs int64 - var expectedMemMiB int64 - for _, it := range typesOut.InstanceTypes { - if aws.StringValue(it.InstanceType) != modifyType { - continue - } - if it.VCpuInfo != nil { - expectedVCPUs = aws.Int64Value(it.VCpuInfo.DefaultVCpus) - } - if it.MemoryInfo != nil { - expectedMemMiB = aws.Int64Value(it.MemoryInfo.SizeInMiB) - } - break - } - require.NotZero(t, expectedVCPUs, "%s missing VCpuInfo.DefaultVCpus", modifyType) - require.NotZero(t, expectedMemMiB, "%s missing MemoryInfo.SizeInMiB", modifyType) - harness.Detail(t, "expected_vcpus", expectedVCPUs, "expected_mem_mib", expectedMemMiB) - - harness.Step(t, "modify-instance-attribute %s type=%s", instanceID, modifyType) - _, err = fix.AWS.EC2.ModifyInstanceAttribute(&ec2.ModifyInstanceAttributeInput{ - InstanceId: aws.String(instanceID), - InstanceType: &ec2.AttributeValue{Value: aws.String(modifyType)}, - }) - require.NoError(t, err, "modify-instance-attribute") - - // Verify describe-instances reflects the new type before we start. - descOut, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "describe-instances") - require.NotEmpty(t, descOut.Reservations, "no reservations for %s", instanceID) - require.NotEmpty(t, descOut.Reservations[0].Instances, "no instances for %s", instanceID) - gotType := aws.StringValue(descOut.Reservations[0].Instances[0].InstanceType) - require.Equalf(t, modifyType, gotType, - "ModifyInstanceAttribute did not stick: want %s got %s", modifyType, gotType) - - harness.Step(t, "start-instances %s", instanceID) - _, err = fix.AWS.EC2.StartInstances(&ec2.StartInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "start-instances") - - runInst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") - - // Re-discover SSH endpoint — qemu hostfwd may have rebound. - host, port := harness.InstancePublicSSHHost(t, runInst) - harness.Detail(t, "ssh_host", host, "ssh_port", port) - - waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} - - harness.Step(t, "ssh nproc") - nprocOut := strings.TrimSpace(runSSH(t, tgt, "nproc")) - vmVCPUs, err := strconv.ParseInt(nprocOut, 10, 64) - require.NoErrorf(t, err, "parse nproc output %q", nprocOut) - require.Equalf(t, expectedVCPUs, vmVCPUs, - "vCPU mismatch after modify: VM=%d expected=%d", vmVCPUs, expectedVCPUs) - harness.Detail(t, "vm_vcpus", vmVCPUs) - - harness.Step(t, "ssh MemTotal") - memOut := strings.TrimSpace(runSSH(t, tgt, "awk '/MemTotal/ {print $2}' /proc/meminfo")) - vmMemKB, err := strconv.ParseInt(memOut, 10, 64) - require.NoErrorf(t, err, "parse MemTotal output %q", memOut) - vmMemMiB := vmMemKB / 1024 - // 15% margin for kernel-reserved memory, matching the bash threshold. - expectedMemLow := expectedMemMiB * 85 / 100 - require.GreaterOrEqualf(t, vmMemMiB, expectedMemLow, - "memory too low after modify: VM=%d MiB expected>=%d MiB (target %d MiB)", - vmMemMiB, expectedMemLow, expectedMemMiB) - harness.Detail(t, "vm_mem_mib", vmMemMiB, "threshold_mib", expectedMemLow) - - // Cleanup restores original type + running state for sibling Test*. -} - -// runRebootInstance starts the (stopped) primary instance, captures its -// pre-reboot private IP, issues a reboot, asserts the API never reports a -// non-running state during a short polling window, waits for SSH to come -// back, and verifies the guest actually rebooted (uptime < 120s) without -// changing its private IP. Leaves the instance running. Maps to -// run-e2e.sh ~1178–1236. -func runRebootInstance(t *testing.T, fix *Fixture) { - harness.Phase(t, "Single — Reboot Running Instance") - - inst, _ := needInstance(t, fix) - instanceID := aws.StringValue(inst.InstanceId) - _, keyPath := needKeyPair(t, fix) - - host, port := harness.InstancePublicSSHHost(t, inst) - waitForSSHReady(t, host, port, keyPath) - - // Capture pre-reboot private IP for the post-reboot identity check. - preDesc, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "describe-instances (pre-reboot)") - require.NotEmpty(t, preDesc.Reservations, "no reservations for %s", instanceID) - require.NotEmpty(t, preDesc.Reservations[0].Instances, "no instances for %s", instanceID) - preRebootIP := aws.StringValue(preDesc.Reservations[0].Instances[0].PrivateIpAddress) - harness.Detail(t, "pre_reboot_private_ip", preRebootIP) - - harness.Step(t, "reboot-instances %s", instanceID) - _, err = fix.AWS.EC2.RebootInstances(&ec2.RebootInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "reboot-instances") - - // Bash polls 10× at 1s checking the state stays "running" — EC2's - // reboot semantics don't transition the instance state at all. - for i := 0; i < 10; i++ { - out, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "describe-instances during reboot poll %d", i) - require.NotEmpty(t, out.Reservations[0].Instances, "instance disappeared during reboot") - state := aws.StringValue(out.Reservations[0].Instances[0].State.Name) - require.Equalf(t, "running", state, - "instance unexpectedly left running state during reboot: %s", state) - time.Sleep(1 * time.Second) - } - - // SSH endpoint may have rebound after the guest restart (qemu hostfwd - // can shift), so re-discover. - descPost, err := fix.AWS.EC2.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "describe-instances (post-reboot)") - require.NotEmpty(t, descPost.Reservations[0].Instances, "no instances post-reboot") - postInst := descPost.Reservations[0].Instances[0] - host, port = harness.InstancePublicSSHHost(t, postInst) - waitForSSHReady(t, host, port, keyPath) - - tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} - harness.Step(t, "ssh uptime") - uptimeOut := strings.TrimSpace(runSSH(t, tgt, "cat /proc/uptime | cut -d. -f1")) - uptimeSecs, err := strconv.ParseInt(uptimeOut, 10, 64) - require.NoErrorf(t, err, "parse uptime output %q", uptimeOut) - require.LessOrEqualf(t, uptimeSecs, int64(120), - "guest uptime %ds is > 120s — reboot may not have occurred", uptimeSecs) - harness.Detail(t, "uptime_secs", uptimeSecs) - - postRebootIP := aws.StringValue(postInst.PrivateIpAddress) - assert.Equalf(t, preRebootIP, postRebootIP, - "PrivateIpAddress changed across reboot: %s -> %s", preRebootIP, postRebootIP) - - // Leave instance running — later phases expect the primary instance to - // still be up. -} - // waitForSSHReady probes a full SSH handshake (BatchMode + ConnectTimeout) // against host:port, retrying until the daemon completes banner exchange. // TCP-port reachability alone is insufficient — sshd accepts the connect @@ -376,7 +176,7 @@ func waitForSSHReady(t *testing.T, host string, port int, keyPath string) { } } -// waitForSSHHandshake is an alias kept for instance_test.go's call site. +// waitForSSHHandshake is an alias kept for guestchurn_test.go's call site. func waitForSSHHandshake(t *testing.T, host string, port int, keyPath string) { waitForSSHReady(t, host, port, keyPath) } diff --git a/tests/e2e/single/tests_test.go b/tests/e2e/single/tests_test.go index 5e75bb4c0..ea603c706 100644 --- a/tests/e2e/single/tests_test.go +++ b/tests/e2e/single/tests_test.go @@ -65,29 +65,10 @@ func TestInstanceMetadata(t *testing.T) { runInstanceMetadata(t, requireSingleNodeFixture(t)) } -func TestSSHProbe(t *testing.T) { - runSSHProbe(t, requireSingleNodeFixture(t)) -} - func TestConsoleOutput(t *testing.T) { runConsoleOutput(t, requireSingleNodeFixture(t)) } -// TestENIHotplug hot-plugs a secondary ENI onto the freshly-booted singleton -// and asserts the NIC reaches the guest, then restores it. Placed before the -// stop/start churn so the singleton is known running + SSH-healthy. -func TestENIHotplug(t *testing.T) { - runENIHotplug(t, requireSingleNodeFixture(t)) -} - -// TestENIHotplugReconcile attaches a secondary ENI, restarts spinifex-daemon, -// and asserts the hot-plug reconciler keeps the ENI attached + manageable -// (detach succeeds) across the restart. Runs after TestENIHotplug so the -// singleton is known running + SSH-healthy. -func TestENIHotplugReconcile(t *testing.T) { - runENIHotplugReconcile(t, requireSingleNodeFixture(t)) -} - func TestVolumeLifecycle(t *testing.T) { runVolumeLifecycle(t, requireSingleNodeFixture(t)) } @@ -96,9 +77,14 @@ func TestVolumeStatus(t *testing.T) { runVolumeStatus(t, requireSingleNodeFixture(t)) } -// TestVolumeDurability stops/starts the singleton; sequential, leaves it running. -func TestVolumeDurability(t *testing.T) { - runVolumeDurability(t, requireSingleNodeFixture(t)) +// TestGuestChurnDurability merges the former TestSSHProbe, TestENIHotplug, +// TestENIHotplugReconcile, TestVolumeDurability, TestModifyInstanceAttribute, +// and TestRebootInstance around one shared guest and one shared +// data-integrity sentinel (see runGuestChurnDurability for the stage +// breakdown and gating rationale). Sequential: it stops/starts and reboots +// the singleton, leaving it running. +func TestGuestChurnDurability(t *testing.T) { + runGuestChurnDurability(t, requireSingleNodeFixture(t)) } func TestSnapshotLifecycle(t *testing.T) { @@ -121,14 +107,6 @@ func TestAttachToStoppedError(t *testing.T) { runAttachToStoppedError(t, requireSingleNodeFixture(t)) } -func TestModifyInstanceAttribute(t *testing.T) { - runModifyInstanceAttribute(t, requireSingleNodeFixture(t)) -} - -func TestRebootInstance(t *testing.T) { - runRebootInstance(t, requireSingleNodeFixture(t)) -} - // --- Sequential: shared-state / sub-test-parallel Tests --- func TestNegativeErrorPaths(t *testing.T) { diff --git a/tests/e2e/single/volume_durability_test.go b/tests/e2e/single/volume_durability_test.go deleted file mode 100644 index ef6e680fb..000000000 --- a/tests/e2e/single/volume_durability_test.go +++ /dev/null @@ -1,100 +0,0 @@ -//go:build e2e - -package single - -import ( - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/mulgadc/spinifex/tests/e2e/harness" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// volumeDataLabel / volumeDataSizeMiB parameterise the durability round-trip. -// A 1 GiB volume is the AWS minimum; the 4 MiB random payload is large enough -// to be a meaningful checksum target while keeping the predastore round-trip -// fast. -const ( - volumeDataLabel = "e2edur" - volumeDataSizeMiB = 4 -) - -// runVolumeDurability proves real guest bytes survive the full -// QEMU↔viperblock↔predastore path across detach/reattach and an instance -// stop/start — the assembled I/O path that no unit test reaches. It also -// asserts CreateVolume reports Encrypted==true, confirming the -// control-plane→viperblockd master-key wiring is live (on-by-default cluster). -// -// Sequential: it stops/starts the shared singleton, leaving it running so -// sibling tests are unaffected (same contract as runStopStart). -func runVolumeDurability(t *testing.T, fix *Fixture) { - harness.Phase(t, "Single — Volume Data Durability Round-Trip") - - az := needAZ(t, fix) - inst, _ := needInstance(t, fix) - instanceID := aws.StringValue(inst.InstanceId) - _, keyPath := needKeyPair(t, fix) - - host, port := harness.InstancePublicSSHHost(t, inst) - waitForSSHReady(t, host, port, keyPath) - tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} - // 1. Create volume + assert the encryption-wiring signal. - harness.Step(t, "create-volume size=1 az=%s", az) - createOut, err := fix.AWS.EC2.CreateVolume(&ec2.CreateVolumeInput{ - AvailabilityZone: aws.String(az), - Size: aws.Int64(1), - }) - require.NoError(t, err, "create-volume") - volID := aws.StringValue(createOut.VolumeId) - require.NotEmpty(t, volID, "CreateVolume returned empty VolumeId") - harness.Detail(t, "volume", volID, "encrypted", aws.BoolValue(createOut.Encrypted)) - assert.Truef(t, aws.BoolValue(createOut.Encrypted), - "CreateVolume.Encrypted should be true (on-by-default cluster master key); "+ - "false means the control-plane→viperblockd key wiring is broken") - harness.RegisterVolumeTeardown(t, fix.AWS, volID) - harness.WaitForVolumeState(t, fix.AWS, volID, "available", harness.WithPoll(500*time.Millisecond)) - - // 2. Attach → format → write sentinel → record sha256. - before := harness.GuestDiskSet(t, tgt) - harness.AttachVolumeWait(t, fix.AWS, volID, instanceID, "/dev/sdf") - dev := harness.WaitForNewGuestDisk(t, tgt, before, 60*time.Second) - harness.Detail(t, "guest_dev", dev) - wantSha := harness.GuestFormatWriteSentinel(t, tgt, dev, volumeDataLabel, volumeDataSizeMiB) - harness.Detail(t, "sha256", wantSha) - - // 3. Detach → reattach → re-read by device; sha must survive. - harness.DetachVolumeWait(t, fix.AWS, volID) - before = harness.GuestDiskSet(t, tgt) - harness.AttachVolumeWait(t, fix.AWS, volID, instanceID, "/dev/sdf") - dev = harness.WaitForNewGuestDisk(t, tgt, before, 60*time.Second) - gotSha := harness.GuestReadSentinelSha(t, tgt, "/dev/"+dev, volumeDataLabel) - require.Equalf(t, wantSha, gotSha, "sha256 mismatch after detach/reattach") - harness.Detail(t, "reattach_sha_ok", gotSha) - - // 4. Stop/start (forces re-mount after instance restart) → re-read by - // label, since the device name can shift across the restart. - harness.Step(t, "stop-instances %s", instanceID) - _, err = fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "stop-instances") - harness.WaitForInstanceState(t, fix.AWS, instanceID, "stopped") - - harness.Step(t, "start-instances %s", instanceID) - _, err = fix.AWS.EC2.StartInstances(&ec2.StartInstancesInput{ - InstanceIds: []*string{aws.String(instanceID)}, - }) - require.NoError(t, err, "start-instances") - runInst := harness.WaitForInstanceState(t, fix.AWS, instanceID, "running") - - // SSH endpoint may have rebound after the restart — re-discover. - host, port = harness.InstancePublicSSHHost(t, runInst) - waitForSSHReady(t, host, port, keyPath) - tgt = harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} - gotSha = harness.GuestReadSentinelSha(t, tgt, "/dev/disk/by-label/"+volumeDataLabel, volumeDataLabel) - require.Equalf(t, wantSha, gotSha, "sha256 mismatch after stop/start") - harness.Detail(t, "stopstart_sha_ok", gotSha) -}