//go:build e2e package single import ( "context" "strings" "testing" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/tests/e2e/harness" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const earlyRebootRecoveryTimeout = 3 * time.Minute // runEarlyRebootLiveness reboots a fresh guest as soon as cloud-init.target is // observable. The control plane must either reject a stuck reset or boot again. func runEarlyRebootLiveness(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — early reboot resumes or reports an API error") instType, _ := needInstanceTypeArch(t, fix) keyName, keyPath := needKeyPair(t, fix) amiID := needAMI(t, fix) vpc := harness.EnsureDefaultVPC(t, fix.Harness) require.NotEmpty(t, vpc.SGID, "default SG ID required") harness.AuthorizeSSHIngress(t, fix.AWS, vpc.SGID) harness.Step(t, "launch fresh guest for early reboot") instanceID := launchBaselineInstance(t, fix, amiID, instType, keyName, vpc.SubnetID, []string{vpc.SGID}) inst := describeSingletonInstance(t, fix, instanceID) host, port := harness.InstancePublicSSHHost(t, inst) sshHealth.WaitReady(t, host, port, keyPath) tgt := harness.SSHTarget{User: "ubuntu", Host: host, Port: port, KeyPath: keyPath} bootID := strings.TrimSpace(harness.RunSSH(t, tgt, "cat /proc/sys/kernel/random/boot_id")) require.NotEmpty(t, bootID, "fresh guest returned an empty boot ID") waitForCloudInitTarget(t, tgt) harness.Step(t, "reboot-instances %s immediately after cloud-init.target", instanceID) _, err := fix.AWS.EC2.RebootInstances(&ec2.RebootInstancesInput{ InstanceIds: []*string{aws.String(instanceID)}, }) if err != nil { var apiErr awserr.Error require.ErrorAs(t, err, &apiErr, "reboot failure must be an AWS API error") assert.Equal(t, "InternalError", apiErr.Code(), "a liveness failure must surface as InternalError") harness.Detail(t, "reboot_outcome", "api_error", "code", apiErr.Code()) return } deadline := time.Now().Add(earlyRebootRecoveryTimeout) // "SSH never answered" and "SSH answered with the boot ID it had before" // are different faults — a guest that never came back versus a reset that // never reached it — and one message for both names neither. var ( reachedOnce bool lastBootID string lastSSHErr error ) for { ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) out, sshErr := harness.RunGuestSSH(ctx, tgt, "cat /proc/sys/kernel/random/boot_id") cancel() lastSSHErr = sshErr if sshErr == nil { reachedOnce = true newBootID := strings.TrimSpace(string(out)) lastBootID = newBootID if newBootID != "" && newBootID != bootID { harness.Detail(t, "reboot_outcome", "resumed", "boot_id", newBootID) return } } if time.Now().After(deadline) { harness.DumpInstanceConsole(t, fix.AWS, instanceID, fix.ArtifactDir(t), "early-reboot-console.log") if !reachedOnce { t.Fatalf("reboot API succeeded but the guest never answered SSH again within %s "+ "(boot ID before the reboot %s, last SSH error: %v)", earlyRebootRecoveryTimeout, bootID, lastSSHErr) } t.Fatalf("reboot API succeeded but the guest answered SSH still holding boot ID %s after %s "+ "(so the reset never reached the guest kernel; last read %q)", bootID, earlyRebootRecoveryTimeout, lastBootID) } time.Sleep(2 * time.Second) } } func waitForCloudInitTarget(t *testing.T, tgt harness.SSHTarget) { t.Helper() deadline := time.Now().Add(60 * time.Second) for { ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) out, err := harness.RunGuestSSH(ctx, tgt, "systemctl is-active cloud-init.target") cancel() if err == nil && strings.TrimSpace(string(out)) == "active" { return } if time.Now().After(deadline) { t.Fatalf("cloud-init.target did not become active within 1m (last output %q, err: %v)", strings.TrimSpace(string(out)), err) } time.Sleep(500 * time.Millisecond) } }