mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
Ben Duncan
ea3e01bb20
* fix: reboot a guest by asking it, not by pulling its reset line RebootInstances was QMP system_reset alone. The guest kernel is never told, so it does not sync, does not unmount and loses every dirty page - which is how a reboot issued seconds after a write loses that write, and how a guest rebooted during cloud-init can fail to come back at all. Guests now launch with -action shutdown=pause, so a clean guest poweroff leaves QEMU paused rather than gone. Reboot presses the power button, waits for the guest to reach that state, then resets and resumes the same process: no volume, tap or state churn, and no window where the guest is missing. A guest that will not go quietly is hard-reset, which is what AWS documents. The button is pressed again every 10s while the guest ignores it. One signal is not a shutdown request on a guest whose acpid has not started, which is the same boot race that produced the failure. Two consequences handled here: a guest that powers itself off now leaves QEMU paused, so the heartbeat quits it; and a guest launched before the flag exits on powerdown rather than pausing, so it is reset without being asked. * fix: answer a reboot when it is accepted, not when the guest is back Asking a guest to shut down takes as long as the guest takes. RebootInstances gives the daemon five seconds to reply, so a reboot that waited for the guest reported a reboot that had in fact succeeded as InvalidInstanceID.NotFound. Manager.Reboot now admits the reboot — the instance exists and is running — and runs the powerdown and reset in the background, which is the contract EC2 documents. The two admission failures are still reported; nothing after them can be, so a reset that fails is logged and left to the heartbeat. The gateway also stops calling a NATS timeout NotFound. Only the absence of a subscriber says anything about where the instance is, which is how the sibling start and terminate paths already read it. * test: prove the churn reboot by its boot ID, not by how fast it happened The subtest waited ten seconds and then read /proc/uptime. A reboot is asynchronous, so the guest answering at that point may be the one that has not shut down yet, and its uptime is whatever it was before — which the test reads as "reboot may not have occurred". Waiting for the boot ID to change is positive proof the guest rebooted and does not depend on how long the guest takes to go down. The uptime assertion stays, now measured against a boot that is known to be new. * test: prove the graceful reboot against real guests and a real engine Two tests the reboot design asked for and the unit tests cannot give. Cell 18 launches two guests that differ only in how they answer the power button. One installs a shutdown hook writing an fsynced line a second, so the file that survives measures how much of its shutdown it was allowed to run — a guest reset instead of asked leaves none of it. The other ignores the power key entirely and must be reset once its budget expires, with the daemon saying so. Each variant asserts the evidence only it can produce, and the slow one asserts the absence of the line the wedged one requires. Cell 27 loads a PostgreSQL instance past its shared buffers with pgbench and reboots it dirty. The probe is an UNLOGGED table: PostgreSQL writes no WAL for one, so a clean shutdown checkpoints it and crash recovery truncates it unconditionally. Rows committed normally survive a hard reset too, so the obvious assertion would pass against the bug. The rds suite and cell 27 both gain budget for the load phase. * fix: stop the process-group test misreading a reaped child as alive The test polled until the child was gone, then checked once more. A task being reaped shows state X for an instant, which processRunning counted as running, so that second look failed about 1 run in 50 under -race. * fix: wait for the recovery retry loop before removing a test's data dir restoreInstances starts the retry loop untracked, and it rewrites the local state file when it ends. A write landing during t.TempDir cleanup failed TestRestoreInstances_* with RemoveAll: directory not empty.
1490 lines
57 KiB
Go
1490 lines
57 KiB
Go
//go:build e2e
|
|
|
|
// Package reboot is the single-node host-reboot resilience suite. It MUST run outside
|
|
// the cluster (on the CI runner) because systemctl reboot kills any in-VM process.
|
|
// AWS API traffic crosses the WAN via the SDK; SSH ops use harness.PeerSSH.
|
|
package reboot
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
|
"github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/aws/aws-sdk-go/service/elbv2"
|
|
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
const (
|
|
keyName = "reboot-e2e-key"
|
|
tgName = "reboot-e2e-tg"
|
|
albName = "reboot-e2e-alb"
|
|
sgName = "reboot-e2e-sg"
|
|
vpcCIDR = "10.210.0.0/16"
|
|
subnetCIDR = "10.210.1.0/24"
|
|
httpPort = 80
|
|
probesPerRun = 20
|
|
appHTTPUnit = "reboot-e2e-httpd.service"
|
|
rebootDoctype = "reboot-e2e"
|
|
)
|
|
|
|
// appUserData provisions a systemd-managed HTTP responder. Using a unit (not nohup) ensures
|
|
// the responder survives the guest's own restart after host reboot.
|
|
const appUserData = `#!/bin/bash
|
|
set -e
|
|
INSTANCE_ID=$(hostname)
|
|
install -d -m 0755 /var/lib/httpd
|
|
echo "{\"instance_id\": \"${INSTANCE_ID}\"}" > /var/lib/httpd/index.html
|
|
cat > /etc/systemd/system/reboot-e2e-httpd.service <<'UNIT'
|
|
[Unit]
|
|
Description=Reboot E2E HTTP responder
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
ExecStart=/usr/bin/python3 -m http.server 80 --bind 0.0.0.0 --directory /var/lib/httpd
|
|
Restart=always
|
|
RestartSec=1
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
UNIT
|
|
systemctl daemon-reload
|
|
systemctl enable --now reboot-e2e-httpd.service
|
|
|
|
# Workaround for viperblock WAL <4MB skip-upload bug:
|
|
# WriteWALToChunk(false) is a no-op when the WAL is under 4MB so small
|
|
# user-data writes (the unit file, index.html) sit in the on-host WAL
|
|
# and are lost when the host reboots before the WAL hits the threshold.
|
|
# Write 8MB of zeros + sync to push WAL over the threshold so the
|
|
# unit-file write definitely lands in predastore before the host reboot.
|
|
# Drop when upstream fix lands.
|
|
dd if=/dev/zero of=/var/lib/spinifex-e2e-pad bs=1M count=8 status=none
|
|
sync
|
|
rm -f /var/lib/spinifex-e2e-pad
|
|
sync
|
|
`
|
|
|
|
type fixture struct {
|
|
env *harness.Env
|
|
artifacts string
|
|
ssh *harness.PeerSSH
|
|
aws *harness.AWSClient
|
|
|
|
instanceType string
|
|
amiID string
|
|
|
|
vpcID string
|
|
subnetID string
|
|
igwID string
|
|
sgID string
|
|
|
|
appInstanceIDs []string
|
|
preIPs map[string]string
|
|
keyPairName string // the pair this fixture created, and the one cleanup removes
|
|
privateKeyPath string // local PEM written from CreateKeyPair KeyMaterial
|
|
|
|
tgArn string
|
|
albArn string
|
|
albID string
|
|
listenerArn string
|
|
albPublicIP string
|
|
albEniMAC string
|
|
|
|
preSwitches int
|
|
prePorts int
|
|
|
|
rebootStart time.Time
|
|
rebootDone bool
|
|
mu sync.Mutex // guards rebootDone for the on-failure dump
|
|
|
|
timeouts timeouts
|
|
}
|
|
|
|
type timeouts struct {
|
|
rebootWait time.Duration
|
|
daemonReady time.Duration
|
|
instanceRunning time.Duration
|
|
lbRecover time.Duration
|
|
}
|
|
|
|
// TestRebootResilience runs the 8-phase reboot resilience sequence.
|
|
// Phases are sequential: the reboot is a global event and each phase depends on the prior.
|
|
func TestRebootResilience(t *testing.T) {
|
|
env := harness.LoadEnv(t)
|
|
requireRunnerMode(t, env)
|
|
|
|
fix := &fixture{
|
|
env: env,
|
|
artifacts: harness.ArtifactDir(t, env),
|
|
ssh: harness.NewPeerSSH(),
|
|
preIPs: map[string]string{},
|
|
timeouts: timeouts{
|
|
rebootWait: durationEnv("REBOOT_WAIT_SECS", 300*time.Second),
|
|
daemonReady: durationEnv("DAEMON_READY_SECS", 180*time.Second),
|
|
instanceRunning: durationEnv("INSTANCE_RUNNING_SECS", 120*time.Second),
|
|
// 5min budget: post-reboot the daemon relaunches app VMs and the ALB sys.micro
|
|
// simultaneously; HC probing starts immediately but targets honestly report
|
|
// unhealthy until the responder unit finishes starting (no cloud-init shortcut).
|
|
lbRecover: durationEnv("LB_RECOVER_SECS", 300*time.Second),
|
|
},
|
|
}
|
|
|
|
resolveTrust(t, fix)
|
|
fix.aws = harness.NewAWSClient(t, env)
|
|
|
|
t.Cleanup(func() { cleanup(t, fix) })
|
|
|
|
harness.OnFailure(t, func() {
|
|
// Dump diagnostics windowed from reboot onwards (if reboot fired) so
|
|
// the analysis surfaces journal lines from the recovery path rather
|
|
// than the per-instance state-transition spam during setup.
|
|
fix.mu.Lock()
|
|
since := fix.rebootStart
|
|
done := fix.rebootDone
|
|
fix.mu.Unlock()
|
|
if !done {
|
|
since = time.Time{}
|
|
}
|
|
dumpDiagnostics(t, fix, since)
|
|
dumpAppDiagnostics(t, fix)
|
|
})
|
|
|
|
harness.Phase(t, "Phase 0 — Prerequisites")
|
|
phase0Prereqs(t, fix)
|
|
|
|
harness.Phase(t, "Phase 1 — VPC + Subnet + Security Group")
|
|
phase1Network(t, fix)
|
|
|
|
harness.Phase(t, "Phase 2 — App instances")
|
|
phase2Instances(t, fix)
|
|
|
|
harness.Phase(t, "Phase 3 — ALB + target group + listener")
|
|
phase3LB(t, fix)
|
|
|
|
harness.Phase(t, "Phase 4 — Pre-reboot traffic")
|
|
runHTTPBurst(t, fix, "ALB pre-reboot")
|
|
|
|
harness.Phase(t, "Phase 5 — Snapshot pre-reboot state")
|
|
phase5Snapshot(t, fix)
|
|
|
|
harness.Phase(t, "Phase 6 — systemctl reboot")
|
|
phase6Reboot(t, fix)
|
|
|
|
harness.Phase(t, "Phase 7 — Wait for spinifex readiness")
|
|
phase7Readiness(t, fix)
|
|
|
|
harness.Phase(t, "Phase 8 — Post-reboot assertions")
|
|
phase8Asserts(t, fix)
|
|
}
|
|
|
|
// --- Setup helpers -------------------------------------------------------
|
|
|
|
// requireRunnerMode bails early if the env isn't runner-resident (SPINIFEX_WAN_IP or endpoint missing).
|
|
func requireRunnerMode(t *testing.T, env *harness.Env) {
|
|
t.Helper()
|
|
if env.WANHost == "" {
|
|
t.Fatalf("reboot suite requires SPINIFEX_WAN_IP (no WAN host resolved)")
|
|
}
|
|
if env.Mode != harness.ModeSingle {
|
|
t.Skipf("reboot suite is single-node only (mode=%s)", env.Mode)
|
|
}
|
|
if endpoint := os.Getenv("SPINIFEX_AWS_ENDPOINT"); endpoint == "" {
|
|
t.Fatalf("reboot suite requires SPINIFEX_AWS_ENDPOINT=https://%s:%d", env.WANHost, env.AWSGWPort)
|
|
}
|
|
}
|
|
|
|
// resolveTrust SCPs the spinifex CA to a tmp file for real TLS verification.
|
|
// Falls back to SPINIFEX_AWS_INSECURE=1 on any error; CA trust is the cert suite's concern.
|
|
func resolveTrust(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
if os.Getenv("SPINIFEX_AWS_INSECURE") == "1" {
|
|
t.Logf("trust: SPINIFEX_AWS_INSECURE=1 set; skipping CA fetch")
|
|
return
|
|
}
|
|
if os.Getenv("SPINIFEX_CA_CERT") != "" {
|
|
t.Logf("trust: SPINIFEX_CA_CERT already set; skipping CA fetch")
|
|
return
|
|
}
|
|
|
|
tmpCA := filepath.Join(fix.artifacts, "spinifex-ca.pem")
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
candidates := []string{
|
|
"/etc/spinifex/ca.pem",
|
|
"$HOME/spinifex/config/ca.pem",
|
|
}
|
|
var lastErr error
|
|
for _, remote := range candidates {
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, "cat "+remote)
|
|
if err == nil && len(out) > 0 {
|
|
if writeErr := os.WriteFile(tmpCA, out, 0o600); writeErr == nil {
|
|
t.Logf("trust: fetched CA from %s -> %s", remote, tmpCA)
|
|
t.Setenv("SPINIFEX_CA_CERT", tmpCA)
|
|
return
|
|
} else {
|
|
lastErr = writeErr
|
|
}
|
|
} else {
|
|
lastErr = err
|
|
}
|
|
}
|
|
t.Logf("trust: CA fetch failed (%v); falling back to InsecureSkipVerify", lastErr)
|
|
t.Setenv("SPINIFEX_AWS_INSECURE", "1")
|
|
}
|
|
|
|
// --- Phases --------------------------------------------------------------
|
|
|
|
func phase0Prereqs(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
harness.Step(t, "WAN_IP=%s", fix.env.WANHost)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer cancel()
|
|
require.NoError(t, fix.ssh.Ping(ctx, fix.env.WANHost), "ssh to %s", fix.env.WANHost)
|
|
harness.Detail(t, "ssh", "ok")
|
|
|
|
fix.instanceType = discoverNanoInstanceType(t, fix.aws)
|
|
harness.Detail(t, "instance_type", fix.instanceType)
|
|
|
|
fix.amiID = discoverAMI(t, fix.aws)
|
|
harness.Detail(t, "ami", fix.amiID)
|
|
|
|
provisionKeyPair(t, fix, keyName)
|
|
}
|
|
|
|
// provisionKeyPair creates a key pair and keeps its material, so a test can SSH
|
|
// into the guests it launches. Recreated rather than reused: the material is
|
|
// returned once, at creation, and a pair left behind by an earlier run cannot
|
|
// be read back.
|
|
func provisionKeyPair(t *testing.T, fix *fixture, name string) {
|
|
t.Helper()
|
|
_, _ = fix.aws.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(name)})
|
|
kpOut, err := fix.aws.EC2.CreateKeyPair(&ec2.CreateKeyPairInput{KeyName: aws.String(name)})
|
|
require.NoErrorf(t, err, "create-key-pair %s", name)
|
|
material := aws.StringValue(kpOut.KeyMaterial)
|
|
require.NotEmptyf(t, material, "create-key-pair %s returned no KeyMaterial", name)
|
|
fix.keyPairName = name
|
|
fix.privateKeyPath = filepath.Join(fix.artifacts, name+".pem")
|
|
require.NoError(t, os.WriteFile(fix.privateKeyPath, []byte(material), 0o600), "write private key")
|
|
harness.Detail(t, "key_pair", name)
|
|
}
|
|
|
|
func phase1Network(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
provisionNetwork(t, fix, netNames{
|
|
sg: sgName,
|
|
sgDesc: "Reboot E2E shared SG (ALB + app instances)",
|
|
vpcCIDR: vpcCIDR,
|
|
subnetCIDR: subnetCIDR,
|
|
})
|
|
}
|
|
|
|
// netNames is the set of names one test's network is built under. Each test in
|
|
// this package brings its own, so tearing one down cannot take the other's VPC.
|
|
type netNames struct {
|
|
sg string
|
|
sgDesc string
|
|
vpcCIDR string
|
|
subnetCIDR string
|
|
}
|
|
|
|
// provisionNetwork builds a VPC, a public subnet and a security group admitting
|
|
// tcp/80 and tcp/22 from anywhere — 22 so the test and its diagnostics can SSH
|
|
// into the guests it launches.
|
|
func provisionNetwork(t *testing.T, fix *fixture, names netNames) {
|
|
t.Helper()
|
|
|
|
vpcOut, err := fix.aws.EC2.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String(names.vpcCIDR)})
|
|
require.NoError(t, err, "create-vpc")
|
|
fix.vpcID = aws.StringValue(vpcOut.Vpc.VpcId)
|
|
harness.Detail(t, "vpc", fix.vpcID)
|
|
|
|
igwOut, err := fix.aws.EC2.CreateInternetGateway(&ec2.CreateInternetGatewayInput{})
|
|
require.NoError(t, err, "create-internet-gateway")
|
|
fix.igwID = aws.StringValue(igwOut.InternetGateway.InternetGatewayId)
|
|
_, err = fix.aws.EC2.AttachInternetGateway(&ec2.AttachInternetGatewayInput{
|
|
InternetGatewayId: aws.String(fix.igwID),
|
|
VpcId: aws.String(fix.vpcID),
|
|
})
|
|
require.NoError(t, err, "attach-internet-gateway")
|
|
harness.Detail(t, "igw", fix.igwID)
|
|
|
|
subnetOut, err := fix.aws.EC2.CreateSubnet(&ec2.CreateSubnetInput{
|
|
VpcId: aws.String(fix.vpcID),
|
|
CidrBlock: aws.String(names.subnetCIDR),
|
|
})
|
|
require.NoError(t, err, "create-subnet")
|
|
fix.subnetID = aws.StringValue(subnetOut.Subnet.SubnetId)
|
|
_, _ = fix.aws.EC2.ModifySubnetAttribute(&ec2.ModifySubnetAttributeInput{
|
|
SubnetId: aws.String(fix.subnetID),
|
|
MapPublicIpOnLaunch: &ec2.AttributeBooleanValue{Value: aws.Bool(true)},
|
|
})
|
|
harness.Detail(t, "subnet", fix.subnetID)
|
|
|
|
sgOut, err := fix.aws.EC2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
|
|
GroupName: aws.String(names.sg),
|
|
Description: aws.String(names.sgDesc),
|
|
VpcId: aws.String(fix.vpcID),
|
|
})
|
|
require.NoError(t, err, "create-security-group")
|
|
fix.sgID = aws.StringValue(sgOut.GroupId)
|
|
|
|
// Use IpPermissions form — vpcd ignores the top-level shorthand field.
|
|
// tcp/22 is opened so on-failure diagnostics can SSH into app guests.
|
|
_, err = fix.aws.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
|
|
GroupId: aws.String(fix.sgID),
|
|
IpPermissions: []*ec2.IpPermission{{
|
|
IpProtocol: aws.String("tcp"),
|
|
FromPort: aws.Int64(httpPort),
|
|
ToPort: aws.Int64(httpPort),
|
|
IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}},
|
|
}, {
|
|
IpProtocol: aws.String("tcp"),
|
|
FromPort: aws.Int64(22),
|
|
ToPort: aws.Int64(22),
|
|
IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}},
|
|
}},
|
|
})
|
|
require.NoError(t, err, "authorize tcp/80 on %s", fix.sgID)
|
|
harness.Detail(t, "sg", fix.sgID)
|
|
}
|
|
|
|
func phase2Instances(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
for i := 0; i < 2; i++ {
|
|
out, err := fix.aws.EC2.RunInstances(&ec2.RunInstancesInput{
|
|
ImageId: aws.String(fix.amiID),
|
|
InstanceType: aws.String(fix.instanceType),
|
|
KeyName: aws.String(keyName),
|
|
SubnetId: aws.String(fix.subnetID),
|
|
SecurityGroupIds: []*string{aws.String(fix.sgID)},
|
|
UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(appUserData))),
|
|
MinCount: aws.Int64(1),
|
|
MaxCount: aws.Int64(1),
|
|
})
|
|
require.NoErrorf(t, err, "run-instances app%d", i+1)
|
|
require.NotEmpty(t, out.Instances)
|
|
id := aws.StringValue(out.Instances[0].InstanceId)
|
|
fix.appInstanceIDs = append(fix.appInstanceIDs, id)
|
|
harness.Detail(t, fmt.Sprintf("app%d", i+1), id)
|
|
}
|
|
|
|
for _, id := range fix.appInstanceIDs {
|
|
harness.WaitForInstanceRunning(t, fix.aws, id, fix.timeouts.instanceRunning)
|
|
}
|
|
|
|
for _, id := range fix.appInstanceIDs {
|
|
ip := describePrivateIP(t, fix.aws, id)
|
|
require.NotEmptyf(t, ip, "%s has no PrivateIpAddress", id)
|
|
fix.preIPs[id] = ip
|
|
harness.Detail(t, id, ip)
|
|
}
|
|
}
|
|
|
|
func phase3LB(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
|
|
tgOut, err := fix.aws.ELBv2.CreateTargetGroup(&elbv2.CreateTargetGroupInput{
|
|
Name: aws.String(tgName),
|
|
Protocol: aws.String("HTTP"),
|
|
Port: aws.Int64(httpPort),
|
|
VpcId: aws.String(fix.vpcID),
|
|
HealthCheckPath: aws.String("/index.html"),
|
|
HealthCheckIntervalSeconds: aws.Int64(5),
|
|
HealthyThresholdCount: aws.Int64(2),
|
|
UnhealthyThresholdCount: aws.Int64(2),
|
|
})
|
|
require.NoError(t, err, "create-target-group")
|
|
fix.tgArn = aws.StringValue(tgOut.TargetGroups[0].TargetGroupArn)
|
|
harness.Detail(t, "tg", fix.tgArn)
|
|
|
|
targets := make([]*elbv2.TargetDescription, len(fix.appInstanceIDs))
|
|
for i, id := range fix.appInstanceIDs {
|
|
targets[i] = &elbv2.TargetDescription{Id: aws.String(id)}
|
|
}
|
|
_, err = fix.aws.ELBv2.RegisterTargets(&elbv2.RegisterTargetsInput{
|
|
TargetGroupArn: aws.String(fix.tgArn),
|
|
Targets: targets,
|
|
})
|
|
require.NoError(t, err, "register-targets")
|
|
|
|
lbOut, err := fix.aws.ELBv2.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
|
|
Name: aws.String(albName),
|
|
Scheme: aws.String("internet-facing"),
|
|
Subnets: []*string{aws.String(fix.subnetID)},
|
|
SecurityGroups: []*string{aws.String(fix.sgID)},
|
|
})
|
|
require.NoError(t, err, "create-load-balancer")
|
|
require.NotEmpty(t, lbOut.LoadBalancers)
|
|
fix.albArn = aws.StringValue(lbOut.LoadBalancers[0].LoadBalancerArn)
|
|
parts := strings.Split(fix.albArn, "/")
|
|
fix.albID = parts[len(parts)-1]
|
|
harness.Detail(t, "alb", fix.albArn)
|
|
|
|
listenerOut, err := fix.aws.ELBv2.CreateListener(&elbv2.CreateListenerInput{
|
|
LoadBalancerArn: aws.String(fix.albArn),
|
|
Protocol: aws.String("HTTP"),
|
|
Port: aws.Int64(httpPort),
|
|
DefaultActions: []*elbv2.Action{{
|
|
Type: aws.String("forward"),
|
|
TargetGroupArn: aws.String(fix.tgArn),
|
|
}},
|
|
})
|
|
require.NoError(t, err, "create-listener")
|
|
fix.listenerArn = aws.StringValue(listenerOut.Listeners[0].ListenerArn)
|
|
|
|
harness.WaitForLBActive(t, fix.aws, fix.albArn, "ALB", 5*time.Minute)
|
|
|
|
eniDesc := fmt.Sprintf("ELB app/%s/%s", albName, fix.albID)
|
|
var eni *ec2.NetworkInterface
|
|
harness.EventuallyErr(t, func() error {
|
|
out, err := fix.aws.EC2.DescribeNetworkInterfaces(&ec2.DescribeNetworkInterfacesInput{
|
|
Filters: []*ec2.Filter{{
|
|
Name: aws.String("description"),
|
|
Values: []*string{aws.String(eniDesc)},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(out.NetworkInterfaces) == 0 {
|
|
return fmt.Errorf("no ENI for %s", eniDesc)
|
|
}
|
|
eni = out.NetworkInterfaces[0]
|
|
return nil
|
|
}, 30*time.Second, 2*time.Second)
|
|
|
|
if eni.Association != nil {
|
|
fix.albPublicIP = aws.StringValue(eni.Association.PublicIp)
|
|
}
|
|
fix.albEniMAC = aws.StringValue(eni.MacAddress)
|
|
require.NotEmpty(t, fix.albPublicIP, "ALB has no public IP")
|
|
harness.Detail(t, "alb_public_ip", fix.albPublicIP, "alb_mac", fix.albEniMAC)
|
|
|
|
harness.WaitForTargetsHealthy(t, fix.aws, fix.tgArn, 2, "ALB", 3*time.Minute)
|
|
}
|
|
|
|
func phase5Snapshot(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, "sudo ovn-nbctl show 2>/dev/null || true")
|
|
if err != nil {
|
|
t.Logf("ovn-nbctl pre-snapshot: %v (continuing with zero counts)", err)
|
|
return
|
|
}
|
|
fix.preSwitches, fix.prePorts = countOVN(string(out))
|
|
harness.Detail(t, "pre_switches", fix.preSwitches, "pre_ports", fix.prePorts)
|
|
harness.DumpFile(t, fix.artifacts, "ovn-pre.txt", out)
|
|
|
|
// Capture OVN chassis identity pre-reboot so the post-reboot dump can be diffed;
|
|
// SB Chassis _uuid churn across reboot orphans every Port_Binding claim.
|
|
cstate, err := fix.ssh.Run(ctx, fix.env.WANHost, ovnChassisStateBundle)
|
|
if err != nil {
|
|
t.Logf("ovn chassis pre-snapshot: %v (non-fatal)", err)
|
|
}
|
|
harness.DumpFile(t, fix.artifacts, "ovn-pre-chassis.txt", cstate)
|
|
}
|
|
|
|
func phase6Reboot(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
fix.mu.Lock()
|
|
fix.rebootStart = time.Now()
|
|
fix.mu.Unlock()
|
|
harness.Step(t, "issuing reboot via ssh (connection will drop)")
|
|
|
|
// SSH dies mid-command when the host shuts down; ignore the error.
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
_, _ = fix.ssh.Run(ctx, fix.env.WANHost, "sudo systemctl reboot")
|
|
|
|
// Wait for shutdown to begin so the pre-reboot SSH window isn't mistaken for "back up".
|
|
time.Sleep(5 * time.Second)
|
|
|
|
harness.Step(t, "polling TCP/22 (timeout %s)", fix.timeouts.rebootWait)
|
|
deadline := time.Now().Add(fix.timeouts.rebootWait)
|
|
var lastErr error
|
|
for {
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("host SSH did not come back within %s (last err: %v)", fix.timeouts.rebootWait, lastErr)
|
|
}
|
|
conn, dialErr := net.DialTimeout("tcp", net.JoinHostPort(fix.env.WANHost, "22"), 5*time.Second)
|
|
if dialErr == nil {
|
|
_ = conn.Close()
|
|
pingCtx, pingCancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
err := fix.ssh.Ping(pingCtx, fix.env.WANHost)
|
|
pingCancel()
|
|
if err == nil {
|
|
break
|
|
}
|
|
lastErr = err
|
|
} else {
|
|
lastErr = dialErr
|
|
}
|
|
time.Sleep(5 * time.Second)
|
|
}
|
|
|
|
fix.mu.Lock()
|
|
fix.rebootDone = true
|
|
elapsed := time.Since(fix.rebootStart).Round(time.Second)
|
|
fix.mu.Unlock()
|
|
harness.Detail(t, "ssh_back_after", elapsed)
|
|
}
|
|
|
|
func phase7Readiness(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
for _, svc := range serviceList {
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, "systemctl is-active "+svc)
|
|
state := strings.TrimSpace(string(out))
|
|
if err != nil && state == "" {
|
|
state = "n/a"
|
|
}
|
|
harness.Detail(t, svc, state)
|
|
}
|
|
|
|
harness.Step(t, "polling describe-instance-types (timeout %s)", fix.timeouts.daemonReady)
|
|
harness.EventuallyErr(t, func() error {
|
|
_, err := fix.aws.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})
|
|
return err
|
|
}, fix.timeouts.daemonReady, 5*time.Second)
|
|
harness.Detail(t, "gateway", "responding")
|
|
}
|
|
|
|
func phase8Asserts(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
|
|
// 8.1 instances back to running
|
|
harness.Step(t, "polling instance state (timeout %s)", fix.timeouts.instanceRunning)
|
|
harness.EventuallyErr(t, func() error {
|
|
for _, id := range fix.appInstanceIDs {
|
|
state := describeInstanceState(t, fix.aws, id)
|
|
if state != "running" {
|
|
return fmt.Errorf("%s state=%s", id, state)
|
|
}
|
|
}
|
|
return nil
|
|
}, fix.timeouts.instanceRunning, 5*time.Second)
|
|
harness.Detail(t, "instances", "all running")
|
|
|
|
// 8.2 private IPs preserved
|
|
for _, id := range fix.appInstanceIDs {
|
|
post := describePrivateIP(t, fix.aws, id)
|
|
assert.Equalf(t, fix.preIPs[id], post, "%s IP drift", id)
|
|
}
|
|
|
|
// 8.3 LaunchTime > REBOOT_START (relaunch signal)
|
|
for _, id := range fix.appInstanceIDs {
|
|
lt := describeLaunchTime(t, fix.aws, id)
|
|
require.Falsef(t, lt.IsZero(), "%s missing LaunchTime", id)
|
|
assert.Truef(t, !lt.Before(fix.rebootStart),
|
|
"%s LaunchTime=%s predates reboot=%s", id, lt, fix.rebootStart)
|
|
}
|
|
|
|
// 8.4 ALB active
|
|
harness.WaitForLBActive(t, fix.aws, fix.albArn, "ALB post-reboot", fix.timeouts.lbRecover)
|
|
|
|
// 8.5 targets healthy — capture diagnostics before failing so a timeout can be
|
|
// pinned to network-level drop vs. app/gateway stall.
|
|
if !pollTargetsHealthy(fix.aws, fix.tgArn, 2, fix.timeouts.lbRecover) {
|
|
harness.Step(t, "ALB post-reboot: targets NOT healthy — capturing diagnostics")
|
|
dctx, dcancel := context.WithTimeout(context.Background(), 120*time.Second)
|
|
dumpHeartbeatDiagnostics(dctx, t, fix)
|
|
dumpOVNDataplane(dctx, t, fix)
|
|
dumpOVNClaimState(dctx, t, fix)
|
|
dumpAppDiagnostics(t, fix)
|
|
dcancel()
|
|
t.Fatalf("ALB post-reboot: targets not healthy within %s (see diag-heartbeat.txt / diag-ovn-flows.txt / diag-ovn-claim.txt / diag-host-net.txt)", fix.timeouts.lbRecover)
|
|
}
|
|
harness.Step(t, "ALB post-reboot: 2 targets healthy")
|
|
|
|
// 8.6 Probe until both backends' instance_ids are seen before asserting round-robin:
|
|
// healthy-cache state may show stale "healthy" until both responders are actually up.
|
|
harness.Step(t, "waiting for ALB to serve HTTP from both backends")
|
|
probeCtx, probeCancel := context.WithTimeout(context.Background(), 180*time.Second)
|
|
defer probeCancel()
|
|
seen := map[string]bool{}
|
|
served := false
|
|
serveDeadline := time.Now().Add(180 * time.Second)
|
|
for time.Now().Before(serveDeadline) {
|
|
out, err := fix.ssh.Run(probeCtx, fix.env.WANHost,
|
|
fmt.Sprintf("curl -s --connect-timeout 2 --max-time 3 'http://%s:%d/' 2>/dev/null", fix.albPublicIP, httpPort))
|
|
if err == nil {
|
|
if id := instanceIDFromResponse(out); id != "" {
|
|
seen[id] = true
|
|
}
|
|
}
|
|
if len(seen) >= 2 {
|
|
served = true
|
|
break
|
|
}
|
|
time.Sleep(1 * time.Second)
|
|
}
|
|
// 8.5 reports targets healthy via the daemon's own view, which can be green
|
|
// while the north-south EIP datapath is dark. Capture dataplane diagnostics on
|
|
// a serve failure too — otherwise this path leaves no flow/ARP evidence.
|
|
if !served {
|
|
harness.Step(t, "ALB serve FAILED — capturing datapath diagnostics")
|
|
dctx, dcancel := context.WithTimeout(context.Background(), 120*time.Second)
|
|
dumpALBProbe(dctx, t, fix)
|
|
dumpOVNDataplane(dctx, t, fix)
|
|
dumpOVNClaimState(dctx, t, fix)
|
|
dumpHeartbeatDiagnostics(dctx, t, fix)
|
|
dumpAppDiagnostics(t, fix)
|
|
dcancel()
|
|
t.Fatalf("ALB did not serve from both backends within 180s post-reboot (saw=%v) "+
|
|
"— see diag-ovn-flows.txt / diag-alb-probe.txt / diag-heartbeat.txt", seen)
|
|
}
|
|
|
|
runHTTPBurst(t, fix, "ALB post-reboot")
|
|
|
|
// 8.6.5 Always capture in-guest responder state (pass or fail) to confirm genuine liveness.
|
|
dumpAppDiagnostics(t, fix)
|
|
|
|
// 8.7 ovn-nbctl drift check
|
|
dumpCtx, dumpCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer dumpCancel()
|
|
out, err := fix.ssh.Run(dumpCtx, fix.env.WANHost, "sudo ovn-nbctl show 2>/dev/null || true")
|
|
if err != nil {
|
|
t.Logf("ovn-nbctl post-snapshot: %v (skipping drift assert)", err)
|
|
return
|
|
}
|
|
harness.DumpFile(t, fix.artifacts, "ovn-post.txt", out)
|
|
postSwitches, postPorts := countOVN(string(out))
|
|
harness.Detail(t, "post_switches", postSwitches, "post_ports", postPorts)
|
|
assert.Equalf(t, fix.preSwitches, postSwitches,
|
|
"ovn switch count drift: pre=%d post=%d", fix.preSwitches, postSwitches)
|
|
assert.Equalf(t, fix.prePorts, postPorts,
|
|
"ovn port count drift: pre=%d post=%d", fix.prePorts, postPorts)
|
|
}
|
|
|
|
// --- HTTP burst ----------------------------------------------------------
|
|
|
|
func runHTTPBurst(t *testing.T, fix *fixture, label string) {
|
|
t.Helper()
|
|
harness.Step(t, "sending %d HTTP requests (%s)", probesPerRun, label)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
cmd := fmt.Sprintf(
|
|
"for i in $(seq 1 %d); do curl -s --max-time 5 'http://%s:%d/' 2>/dev/null; echo; done",
|
|
probesPerRun, fix.albPublicIP, httpPort,
|
|
)
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, cmd)
|
|
if err != nil {
|
|
t.Logf("%s: ssh burst error: %v", label, err)
|
|
}
|
|
|
|
counts := map[string]int{}
|
|
totalOK := 0
|
|
for _, line := range strings.Split(string(out), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
var payload struct {
|
|
InstanceID string `json:"instance_id"`
|
|
}
|
|
if jsonErr := json.Unmarshal([]byte(line), &payload); jsonErr != nil || payload.InstanceID == "" {
|
|
continue
|
|
}
|
|
counts[payload.InstanceID]++
|
|
totalOK++
|
|
}
|
|
|
|
for id, n := range counts {
|
|
harness.Detail(t, id, fmt.Sprintf("%d responses", n))
|
|
}
|
|
assert.GreaterOrEqualf(t, len(counts), 2,
|
|
"%s round-robin: expected >=2 unique responders, got %d (counts=%v)", label, len(counts), counts)
|
|
assert.GreaterOrEqualf(t, totalOK, probesPerRun/2,
|
|
"%s success rate: only %d/%d", label, totalOK, probesPerRun)
|
|
}
|
|
|
|
// --- Discovery -----------------------------------------------------------
|
|
|
|
func discoverNanoInstanceType(t *testing.T, c *harness.AWSClient) string {
|
|
t.Helper()
|
|
out, err := c.EC2.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{})
|
|
require.NoError(t, err, "describe-instance-types")
|
|
for _, it := range out.InstanceTypes {
|
|
name := aws.StringValue(it.InstanceType)
|
|
if strings.Contains(name, "nano") {
|
|
return name
|
|
}
|
|
}
|
|
t.Fatal("no nano instance type available")
|
|
return ""
|
|
}
|
|
|
|
// discoverAMI prefers ubuntu, then non-alpine, then anything — matches
|
|
// run-reboot-e2e.sh's three-stage AMI fallback chain.
|
|
func discoverAMI(t *testing.T, c *harness.AWSClient) string {
|
|
t.Helper()
|
|
out, err := c.EC2.DescribeImages(&ec2.DescribeImagesInput{})
|
|
require.NoError(t, err, "describe-images")
|
|
var ubuntu, nonAlpine, anyAMI string
|
|
for _, img := range out.Images {
|
|
id := aws.StringValue(img.ImageId)
|
|
name := aws.StringValue(img.Name)
|
|
if anyAMI == "" {
|
|
anyAMI = id
|
|
}
|
|
if !strings.Contains(strings.ToLower(name), "alpine") && nonAlpine == "" {
|
|
nonAlpine = id
|
|
}
|
|
if strings.HasPrefix(name, "ami-ubuntu") {
|
|
ubuntu = id
|
|
break
|
|
}
|
|
}
|
|
for _, candidate := range []string{ubuntu, nonAlpine, anyAMI} {
|
|
if candidate != "" {
|
|
return candidate
|
|
}
|
|
}
|
|
t.Fatal("no AMIs available")
|
|
return ""
|
|
}
|
|
|
|
func describePrivateIP(t *testing.T, c *harness.AWSClient, id string) string {
|
|
t.Helper()
|
|
out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
|
InstanceIds: []*string{aws.String(id)},
|
|
})
|
|
if err != nil {
|
|
t.Logf("describe %s: %v", id, err)
|
|
return ""
|
|
}
|
|
if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 {
|
|
return ""
|
|
}
|
|
return aws.StringValue(out.Reservations[0].Instances[0].PrivateIpAddress)
|
|
}
|
|
|
|
func describePublicIP(t *testing.T, c *harness.AWSClient, id string) string {
|
|
t.Helper()
|
|
out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
|
InstanceIds: []*string{aws.String(id)},
|
|
})
|
|
if err != nil {
|
|
t.Logf("describe %s: %v", id, err)
|
|
return ""
|
|
}
|
|
if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 {
|
|
return ""
|
|
}
|
|
return aws.StringValue(out.Reservations[0].Instances[0].PublicIpAddress)
|
|
}
|
|
|
|
func describeInstanceState(t *testing.T, c *harness.AWSClient, id string) string {
|
|
t.Helper()
|
|
out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
|
InstanceIds: []*string{aws.String(id)},
|
|
})
|
|
if err != nil {
|
|
return "missing"
|
|
}
|
|
if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 {
|
|
return "missing"
|
|
}
|
|
return aws.StringValue(out.Reservations[0].Instances[0].State.Name)
|
|
}
|
|
|
|
func describeLaunchTime(t *testing.T, c *harness.AWSClient, id string) time.Time {
|
|
t.Helper()
|
|
out, err := c.EC2.DescribeInstances(&ec2.DescribeInstancesInput{
|
|
InstanceIds: []*string{aws.String(id)},
|
|
})
|
|
if err != nil {
|
|
return time.Time{}
|
|
}
|
|
if len(out.Reservations) == 0 || len(out.Reservations[0].Instances) == 0 {
|
|
return time.Time{}
|
|
}
|
|
lt := out.Reservations[0].Instances[0].LaunchTime
|
|
if lt == nil {
|
|
return time.Time{}
|
|
}
|
|
return *lt
|
|
}
|
|
|
|
// --- OVN parsing ---------------------------------------------------------
|
|
|
|
// countOVN extracts logical-switch and port counts from `ovn-nbctl show`
|
|
// output by line-prefix matching (matches the bash `grep -c '^switch '` and
|
|
// `grep -c '^ port '` checks 1:1).
|
|
func countOVN(out string) (switches, ports int) {
|
|
for _, line := range strings.Split(out, "\n") {
|
|
switch {
|
|
case strings.HasPrefix(line, "switch "):
|
|
switches++
|
|
case strings.HasPrefix(line, " port "):
|
|
ports++
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
func instanceIDFromResponse(out []byte) string {
|
|
var payload struct {
|
|
InstanceID string `json:"instance_id"`
|
|
}
|
|
if err := json.Unmarshal(out, &payload); err != nil {
|
|
return ""
|
|
}
|
|
return payload.InstanceID
|
|
}
|
|
|
|
// --- Diagnostics ---------------------------------------------------------
|
|
|
|
var serviceList = []string{
|
|
"spinifex-nats", "spinifex-predastore", "spinifex-viperblock",
|
|
"spinifex-daemon", "spinifex-awsgw", "spinifex-vpcd",
|
|
"ovn-controller", "ovs-vswitchd", "ovn-northd",
|
|
}
|
|
|
|
func dumpDiagnostics(t *testing.T, fix *fixture, since time.Time) {
|
|
t.Helper()
|
|
var modeFlag string
|
|
if since.IsZero() {
|
|
modeFlag = "-n 200"
|
|
harness.Step(t, "diagnostics: last 200 lines per service")
|
|
} else {
|
|
modeFlag = fmt.Sprintf("--since=@%d", since.Unix())
|
|
harness.Step(t, "diagnostics: since reboot @%d", since.Unix())
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
|
defer cancel()
|
|
|
|
if !fix.ssh.IsReachable(ctx, fix.env.WANHost) {
|
|
harness.Step(t, "diagnostics: node unreachable; skipping journal dump")
|
|
return
|
|
}
|
|
|
|
for _, svc := range serviceList {
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost,
|
|
fmt.Sprintf("sudo journalctl -u %s --no-pager %s 2>/dev/null || true", svc, modeFlag))
|
|
if err != nil {
|
|
harness.Detail(t, svc, fmt.Sprintf("journal err: %v", err))
|
|
continue
|
|
}
|
|
harness.DumpFile(t, fix.artifacts, fmt.Sprintf("journal-%s.log", svc), out)
|
|
}
|
|
}
|
|
|
|
// dumpAppDiagnostics probes each app instance's public IP from the host.
|
|
// App VMs have no QEMU hostfwd, so public-IP probes are the only available path.
|
|
func dumpAppDiagnostics(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
if len(fix.appInstanceIDs) == 0 {
|
|
return
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
|
defer cancel()
|
|
|
|
if !fix.ssh.IsReachable(ctx, fix.env.WANHost) {
|
|
harness.Step(t, "app diagnostics: host unreachable; skipping")
|
|
return
|
|
}
|
|
|
|
harness.Step(t, "app diagnostics: probing %d app instance(s)", len(fix.appInstanceIDs))
|
|
|
|
dumpHostNetState(ctx, t, fix)
|
|
dumpALBProbe(ctx, t, fix)
|
|
|
|
for _, id := range fix.appInstanceIDs {
|
|
diagnoseAppInstance(ctx, t, fix, id)
|
|
}
|
|
}
|
|
|
|
// dumpHostNetState captures OVN/OVS/route state on the host in a single round-trip.
|
|
func dumpHostNetState(ctx context.Context, t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
bundle := `set +e
|
|
echo "=== ip route show ==="
|
|
ip route show 2>&1
|
|
echo "=== ip route show 192.168.0.0/24 ==="
|
|
ip route show 192.168.0.0/24 2>&1
|
|
echo "=== ip neigh show dev br-wan ==="
|
|
ip neigh show dev br-wan 2>&1 || ip neigh show 2>&1
|
|
echo "=== ovn-nbctl list nat ==="
|
|
sudo ovn-nbctl list nat 2>&1
|
|
echo "=== ovn-nbctl show ==="
|
|
sudo ovn-nbctl show 2>&1
|
|
echo "=== ovs-vsctl show ==="
|
|
sudo ovs-vsctl show 2>&1
|
|
echo "=== ovs-vsctl list-ports br-int ==="
|
|
sudo ovs-vsctl list-ports br-int 2>&1
|
|
echo "=== ip -d link show veth-wan-ovs (admin/carrier state) ==="
|
|
ip -d link show veth-wan-ovs 2>&1
|
|
echo "=== ip -d link show veth-wan-br (admin/carrier state) ==="
|
|
ip -d link show veth-wan-br 2>&1
|
|
echo "=== ovs-ofctl show br-ext (ofport numbering) ==="
|
|
sudo ovs-ofctl show br-ext 2>&1
|
|
`
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, bundle)
|
|
harness.DumpFile(t, fix.artifacts, "diag-host-net.txt", out)
|
|
if err != nil {
|
|
harness.Detail(t, "host-net", fmt.Sprintf("dump err: %v", err))
|
|
}
|
|
}
|
|
|
|
// pollTargetsHealthy is the non-fatal twin of harness.WaitForTargetsHealthy:
|
|
// returns true on healthy, false on timeout, so the caller can capture diagnostics before aborting.
|
|
func pollTargetsHealthy(c *harness.AWSClient, tgArn string, expected int, timeout time.Duration) bool {
|
|
deadline := time.Now().Add(timeout)
|
|
for {
|
|
out, err := c.ELBv2.DescribeTargetHealth(&elbv2.DescribeTargetHealthInput{
|
|
TargetGroupArn: aws.String(tgArn),
|
|
})
|
|
if err == nil {
|
|
healthy := 0
|
|
for _, th := range out.TargetHealthDescriptions {
|
|
if aws.StringValue(th.TargetHealth.State) == "healthy" {
|
|
healthy++
|
|
}
|
|
}
|
|
if healthy >= expected {
|
|
return true
|
|
}
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return false
|
|
}
|
|
time.Sleep(5 * time.Second)
|
|
}
|
|
}
|
|
|
|
// dumpHeartbeatDiagnostics captures whether the lb-agent's heartbeat to the gateway is
|
|
// failing at the network layer or the app/gateway layer. Tcpdumps both the gateway port
|
|
// (any iface) and the br-wan north-south path; also pulls lb-agent console and awsgw journal.
|
|
func dumpHeartbeatDiagnostics(ctx context.Context, t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
if fix.albPublicIP == "" {
|
|
harness.Detail(t, "heartbeat", "no ALB public IP captured")
|
|
return
|
|
}
|
|
bundle := fmt.Sprintf(`set +e
|
|
echo "=== tcpdump ANY iface, gateway port :%[2]d (real heartbeat path), 14s ==="
|
|
sudo timeout 14 tcpdump -ni any -c 80 "tcp port %[2]d" 2>&1
|
|
echo "=== tcpdump br-wan ALB floating IP %[1]s + ARP (north-south), 8s ==="
|
|
sudo timeout 8 tcpdump -ni br-wan -c 40 "host %[1]s or (arp and host %[1]s)" 2>&1
|
|
echo "=== neigh for ALB floating IP %[1]s ==="
|
|
ip neigh show %[1]s 2>&1 || ip neigh show 2>&1 | grep -F %[1]s
|
|
echo "=== conntrack for gateway port %[2]d ==="
|
|
sudo conntrack -L 2>/dev/null | grep -E "dport=%[2]d|sport=%[2]d" | head || echo "(no conntrack entries / tool absent)"
|
|
echo "=== lb-agent console: Heartbeat / Config / Agent lines ==="
|
|
sudo grep -aE "Heartbeat|Config (hash|applied)|Agent started" /run/spinifex/console-*.log 2>/dev/null | tail -25
|
|
echo "=== awsgw journal: did the gateway RECEIVE heartbeats? ==="
|
|
sudo journalctl -u spinifex-awsgw --no-pager --since "3 min ago" 2>/dev/null | grep -iE "heartbeat|lb-agent|/lb/|target.?health" | tail -20
|
|
`, fix.albPublicIP, fix.env.AWSGWPort)
|
|
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, bundle)
|
|
harness.DumpFile(t, fix.artifacts, "diag-heartbeat.txt", out)
|
|
if err != nil {
|
|
harness.Detail(t, "heartbeat", fmt.Sprintf("dump err: %v", err))
|
|
return
|
|
}
|
|
body := string(out)
|
|
// Detect any L4 packet on the gateway port.
|
|
sawGatewayL4 := strings.Contains(body, fmt.Sprintf(".%d:", fix.env.AWSGWPort)) ||
|
|
strings.Contains(body, fmt.Sprintf(".%d >", fix.env.AWSGWPort)) ||
|
|
strings.Contains(body, fmt.Sprintf("> %s.%d", fix.env.WANHost, fix.env.AWSGWPort))
|
|
switch {
|
|
case sawGatewayL4:
|
|
harness.Detail(t, "heartbeat", "gateway-port PACKETS FLOW — network path healthy; failure is app/gateway-layer (TLS/HTTP/gateway state)")
|
|
default:
|
|
harness.Detail(t, "heartbeat", "NO packets on gateway port — network-level drop; check diag-ovn-flows.txt for br-int forwarding")
|
|
}
|
|
}
|
|
|
|
// dumpOVNDataplane captures OVN/OVS dataplane state at failure time: topology, br-int flows,
|
|
// Port_Bindings, NAT rows, neigh table, and ALB floating-IP reachability from the host.
|
|
func dumpOVNDataplane(ctx context.Context, t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
bundle := fmt.Sprintf(`set +e
|
|
EIP=%[1]s
|
|
echo "=== ovs-vsctl show (bridges + ports) ==="
|
|
sudo ovs-vsctl show 2>&1 | head -120
|
|
echo "=== ovs-ofctl show br-int (port->ofport map) ==="
|
|
sudo ovs-ofctl show br-int 2>&1 | grep -aE '\(' | head -50
|
|
echo "=== br-int flows: DNAT/conntrack (ct_/nat/ct) ==="
|
|
sudo ovs-ofctl dump-flows br-int 2>&1 | grep -aE 'ct_|nat\(|ct\(' | head -120
|
|
echo "=== br-int flows: EIP + guest subnet 10.210 (forwarding) ==="
|
|
sudo ovs-ofctl dump-flows br-int 2>&1 | grep -aE "nw_dst=$EIP|10\.210\.1\.|192\.168\.0\.21" | head -150
|
|
echo "=== br-int flows: terminal output: actions ==="
|
|
sudo ovs-ofctl dump-flows br-int 2>&1 | grep -aE 'output:|resubmit' | grep -avE 'table=(0|8|9|10|11),' | head -60
|
|
echo "=== ovs-ofctl dump-flows br-ext ==="
|
|
sudo ovs-ofctl dump-flows br-ext 2>&1 | head -120
|
|
echo "=== ovs-appctl fdb/show br-ext (WAN-side MAC learning) ==="
|
|
sudo ovs-appctl fdb/show br-ext 2>&1 | head -40
|
|
echo "=== ovn-sbctl Port_Binding (chassis assignment) ==="
|
|
sudo ovn-sbctl --no-leader-only list Port_Binding 2>&1 | grep -E "logical_port|chassis|mac|type " | head -400
|
|
echo "=== DGP / chassisredirect claim state (untruncated) ==="
|
|
echo "--- every chassisredirect Port_Binding -> resident chassis ---"
|
|
for cr in $(sudo ovn-sbctl --no-leader-only --bare --columns=logical_port find Port_Binding type=chassisredirect 2>/dev/null); do
|
|
ch=$(sudo ovn-sbctl --no-leader-only --bare --columns=chassis find Port_Binding logical_port="$cr" 2>/dev/null)
|
|
hg=$(sudo ovn-sbctl --no-leader-only --bare --columns=ha_chassis_group find Port_Binding logical_port="$cr" 2>/dev/null)
|
|
echo " CR=[$cr] chassis=[$ch] ha_chassis_group=[$hg]"
|
|
done
|
|
echo "--- every gw-vpc LRP -> its referenced chassis-redirect-port (does the CR row exist?) ---"
|
|
for gp in $(sudo ovn-sbctl --no-leader-only --bare --columns=logical_port find Port_Binding 2>/dev/null | tr ' ' '\n' | grep -aE '^gw-vpc'); do
|
|
crp=$(sudo ovn-sbctl --no-leader-only --bare --columns=options find Port_Binding logical_port="$gp" 2>/dev/null | tr ',' '\n' | grep -a chassis-redirect-port | cut -d= -f2)
|
|
exists=$(sudo ovn-sbctl --no-leader-only --bare --columns=_uuid find Port_Binding logical_port="$crp" 2>/dev/null)
|
|
echo " GW-LRP=[$gp] redirect-port=[$crp] CR_row_exists=[$exists]"
|
|
done
|
|
echo "--- NB Gateway_Chassis bindings (DGP HA) ---"
|
|
sudo ovn-nbctl --no-leader-only list Gateway_Chassis 2>&1 | grep -aE 'name|chassis_name|priority' | head -60
|
|
echo "--- NB Logical_Router_Port (name + gateway_chassis + networks) ---"
|
|
sudo ovn-nbctl --no-leader-only --columns=name,gateway_chassis,networks list Logical_Router_Port 2>&1 | head -80
|
|
echo "=== ovn-nbctl dnat_and_snat NAT rows ==="
|
|
sudo ovn-nbctl --no-leader-only list NAT 2>&1 | grep -E "external_ip|logical_ip|external_mac|type " | head -60
|
|
echo "=== host neigh (all floating IPs) ==="
|
|
ip neigh show 2>&1 | grep -E "br-wan|br-int" | head -40
|
|
echo "=== REAL ARP to EIP $EIP (flush primed neigh first) ==="
|
|
sudo ip neigh flush $EIP 2>&1 || true
|
|
ping -c 2 -W 2 $EIP 2>&1
|
|
ip neigh show $EIP 2>&1
|
|
GWMAC=$(ip neigh show $EIP | awk '{for(i=1;i<=NF;i++) if($i=="lladdr") print $(i+1)}')
|
|
echo "discovered cr-gw MAC for EIP: [$GWMAC]"
|
|
echo "=== ofproto/trace north-south DNAT (real ingress per br-int patch port, dl_dst=cr-gw) ==="
|
|
if [ -n "$GWMAC" ]; then
|
|
for p in $(sudo ovs-vsctl list-ports br-int 2>/dev/null | grep -aE 'patch-br-int-to-ext'); do
|
|
echo "--- in_port=$p ---"
|
|
sudo ovs-appctl ofproto/trace br-int "in_port=$p,tcp,dl_src=02:00:00:00:00:01,dl_dst=$GWMAC,nw_src=192.168.0.11,nw_dst=$EIP,nw_ttl=64,tp_src=40000,tp_dst=80" 2>&1 | head -200
|
|
done
|
|
else
|
|
echo "(no cr-gw MAC resolved — ARP itself failed)"
|
|
fi
|
|
echo "=== south->north RETURN trace, unfiltered (guest reply -> un-DNAT -> WAN) ==="
|
|
PRIV=$(sudo ovn-nbctl --no-leader-only --bare --columns=logical_ip find NAT external_ip="\"$EIP\"" 2>/dev/null | head -1)
|
|
CLIENT=$({ sudo ovs-appctl dpctl/dump-conntrack 2>/dev/null || sudo conntrack -L 2>/dev/null; } | grep -aE "dst=$EIP" | grep -aoE 'src=[0-9.]+' | head -1 | cut -d= -f2)
|
|
[ -z "$CLIENT" ] && CLIENT=192.168.1.1
|
|
GLSP=""; GMAC=""
|
|
for lp in $(sudo ovn-sbctl --no-leader-only --bare --columns=logical_port find Port_Binding type='""' 2>/dev/null); do
|
|
m=$(sudo ovn-sbctl --no-leader-only --bare --columns=mac find Port_Binding logical_port="$lp" 2>/dev/null)
|
|
case "$m" in *"$PRIV"*) GLSP="$lp"; GMAC=$(echo "$m" | awk '{print $1}'); break;; esac
|
|
done
|
|
TAP=$(sudo ovs-vsctl --bare --columns=name find Interface external_ids:iface-id="$GLSP" 2>/dev/null | head -1)
|
|
SUBNET_GW=$(echo "$PRIV" | awk -F. '{print $1"."$2"."$3".1"}')
|
|
LRPMAC=$(sudo ovs-ofctl dump-flows br-int 2>/dev/null | grep -a "arp_tpa=$SUBNET_GW," | grep -aoE 'mod_dl_src:[0-9a-f:]+' | head -1 | cut -d: -f2-)
|
|
echo "PRIV=[$PRIV] CLIENT=[$CLIENT] GLSP=[$GLSP] GMAC=[$GMAC] TAP=[$TAP] LRPMAC=[$LRPMAC]"
|
|
if [ -n "$TAP" ] && [ -n "$GMAC" ] && [ -n "$LRPMAC" ]; then
|
|
sudo ovs-appctl ofproto/trace br-int "in_port=$TAP,tcp,dl_src=$GMAC,dl_dst=$LRPMAC,nw_src=$PRIV,nw_dst=$CLIENT,nw_ttl=64,tp_src=80,tp_dst=40000,tcp_flags=ack" 2>&1 | head -300
|
|
else
|
|
echo "(return-trace discovery incomplete — skipping)"
|
|
fi
|
|
echo "=== conntrack DNAT entries (EIP/guest) ==="
|
|
{ sudo ovs-appctl dpctl/dump-conntrack 2>/dev/null || sudo conntrack -L 2>/dev/null; } | grep -aE "$EIP|10\.210\.1\." | head -30 || echo "(none)"
|
|
echo "=== tap capture: does the SYN reach the guest? (tcpdump -i any while curling EIP) ==="
|
|
sudo timeout 6 tcpdump -i any -nn -c 25 "host $EIP or net 10.210.0.0/16" 2>&1 &
|
|
TCPID=$!
|
|
sleep 1
|
|
curl -s --connect-timeout 2 --max-time 4 "http://$EIP/" >/dev/null 2>&1 &
|
|
wait $TCPID 2>/dev/null
|
|
echo "=== (end tap capture) ==="
|
|
echo "=== datapath megaflows carrying the guest reply (what the kernel actually did, not a trace) ==="
|
|
sudo ovs-appctl dpctl/dump-flows -m 2>/dev/null | grep -aF "src=$PRIV" | head -30 || echo "(none)"
|
|
echo "=== PERSISTED-STATE DUMP (survives ovn-controller restart) ==="
|
|
echo "--- external next-hop MACs (regenerated each boot) ---"
|
|
for IF in veth-wan-ovs veth-wan-br br-wan br-ext; do
|
|
M=$(ip link show "$IF" 2>/dev/null | grep -aoE 'link/ether [0-9a-f:]+' | awk '{print $2}')
|
|
echo " $IF mac=[$M]"
|
|
done
|
|
echo "--- SB MAC_Binding (logical_port ip mac) ---"
|
|
sudo ovn-sbctl --no-leader-only list MAC_Binding 2>&1 | grep -aE 'ip|mac|logical_port' | head -40 || echo "(none)"
|
|
echo "--- host neigh on external subnet ---"
|
|
ip neigh show 2>/dev/null | grep -aE '192\.168\.' | head -20 || echo "(none)"
|
|
echo "=== live failure marker (confirms drop is active during dump) ==="
|
|
curl -s -o /dev/null -w "eip-probe HTTP=%%{http_code} time=%%{time_total}\n" --connect-timeout 2 --max-time 5 "http://$EIP/" 2>&1
|
|
echo "=== NB-SOURCE OF THE RETURN-PATH DROP (names what installs table-79 drop) ==="
|
|
echo "--- SB lflow stages that DROP the guest reply (match nw_src=$PRIV) ---"
|
|
sudo ovn-sbctl --no-leader-only lflow-list 2>/dev/null | grep -aiE 'drop' | grep -aF "$PRIV" | head -20 || echo "(none — drop is not a northd logical flow; likely ovn-controller port-security)"
|
|
echo "--- SB lflow lines mentioning guest MAC $GMAC (stage + match context) ---"
|
|
sudo ovn-sbctl --no-leader-only lflow-list 2>/dev/null | grep -aiF "$GMAC" | head -20 || echo "(none)"
|
|
echo "--- NB NAT rows: type/external_mac/logical_port/options (centralised=[] vs distributed) ---"
|
|
sudo ovn-nbctl --no-leader-only --columns=external_ip,logical_ip,type,external_mac,logical_port,options list NAT 2>/dev/null | head -60 || echo "(none)"
|
|
echo "--- NB Logical_Router_Policy (drop-gate + per-instance reroute) ---"
|
|
sudo ovn-nbctl --no-leader-only --columns=priority,match,action,nexthop list Logical_Router_Policy 2>/dev/null | head -40 || echo "(none)"
|
|
echo "--- NB LSP addresses + port_security (source if drop is physical port-sec) ---"
|
|
sudo ovn-nbctl --no-leader-only --columns=name,addresses,port_security list Logical_Switch_Port 2>/dev/null | grep -aF -A0 "$PRIV" | head -20 || echo "(none)"
|
|
echo "=== (end NB-source dump) ==="
|
|
echo "=== ATTRIBUTION OF ANY DROP CARRYING THE GUEST SOURCE (OF cookie -> SB Logical_Flow) ==="
|
|
echo "--- note: OVN table 79 is MAC-binding accounting; its drop is a counted clone, not a loss ---"
|
|
DROPS=$(sudo ovs-ofctl dump-flows br-int 2>/dev/null | grep -a 'actions=drop' | grep -aF "nw_src=$PRIV")
|
|
echo "${DROPS:-(none)}" | head -20
|
|
LF=$(mktemp)
|
|
sudo ovn-sbctl --no-leader-only --columns=_uuid,pipeline,table_id,priority,match,actions \
|
|
list Logical_Flow >"$LF" 2>/dev/null
|
|
for C in $(echo "$DROPS" | grep -aoE 'cookie=0x[0-9a-f]+' | cut -d= -f2 | sort -u); do
|
|
P=$(printf '%%08x' "$C" 2>/dev/null)
|
|
echo "--- cookie $C is Logical_Flow $P* ---"
|
|
awk -v p="$P" 'BEGIN{RS=""} index($0,p)' "$LF" | head -12
|
|
grep -qaF "$P" "$LF" || echo " (no SB lflow — installed by ovn-controller itself, not northd)"
|
|
done
|
|
rm -f "$LF"
|
|
echo "--- is_chassis_resident inputs: bindings this ovn-controller claims locally ---"
|
|
sudo ovn-appctl -t ovn-controller debug/dump-local-bindings 2>&1 | head -40
|
|
echo "--- Port_Binding for guest LSP $GLSP (chassis must be this chassis) ---"
|
|
sudo ovn-sbctl --no-leader-only list Port_Binding "$GLSP" 2>&1 \
|
|
| grep -aE '_uuid|chassis|logical_port|^up|requested' | head -20
|
|
echo "=== (end drop attribution) ==="
|
|
`, fix.albPublicIP)
|
|
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, bundle)
|
|
harness.DumpFile(t, fix.artifacts, "diag-ovn-flows.txt", out)
|
|
if err != nil {
|
|
harness.Detail(t, "ovn", fmt.Sprintf("dump err: %v", err))
|
|
return
|
|
}
|
|
body := string(out)
|
|
reachable := strings.Contains(body, " 0% packet loss")
|
|
hasFlows := strings.Contains(body, "cookie=") && strings.Contains(body, "table=")
|
|
switch {
|
|
case reachable:
|
|
harness.Detail(t, "ovn", "ALB floating IP reachable from host — forwarding OK, failure is elsewhere")
|
|
case hasFlows:
|
|
harness.Detail(t, "ovn", "br-int has flows but floating IP unreachable — flow/forwarding mismatch (inspect dump-flows + ofproto/trace)")
|
|
default:
|
|
harness.Detail(t, "ovn", "br-int flows MISSING and floating IP unreachable — dataplane flow-restore gap post-reboot")
|
|
}
|
|
}
|
|
|
|
// ovnChassisStateBundle captures OVN chassis identity + SB records for pre/post diff.
|
|
// _uuid churn across reboot orphans every Port_Binding claim.
|
|
const ovnChassisStateBundle = `set +e
|
|
echo "=== OVS system-id (chassis identity; must be stable across reboot) ==="
|
|
sudo cat /etc/openvswitch/system-id.conf 2>&1
|
|
sudo ovs-vsctl get Open_vSwitch . external_ids:system-id 2>&1
|
|
echo "=== ovn-sbctl list Chassis (name/_uuid/hostname/encaps) ==="
|
|
sudo ovn-sbctl --no-leader-only list Chassis 2>&1 | grep -E "_uuid|name|hostname|ip " | head -40
|
|
echo "=== ovn-sbctl list Chassis_Private (claimed-by + nb_cfg) ==="
|
|
sudo ovn-sbctl --no-leader-only list Chassis_Private 2>&1 | grep -E "_uuid|name|chassis|nb_cfg" | head -40
|
|
echo "=== ovn get-connection (did set-connection persist?) ==="
|
|
sudo ovn-nbctl get-connection 2>&1
|
|
sudo ovn-sbctl get-connection 2>&1
|
|
`
|
|
|
|
// dumpOVNClaimState captures evidence for why ovn-controller is not claiming Port_Bindings.
|
|
// Pulls /var/log/ovn directly (not journald), SB connection state, and boot ordering
|
|
// to distinguish chassis-identity churn from an SB/NB startup-ordering race.
|
|
func dumpOVNClaimState(ctx context.Context, t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
bundle := ovnChassisStateBundle + `echo "=== ovn-controller SB connection-status ==="
|
|
sudo env OVS_RUNDIR=/var/run/ovn ovs-appctl -t ovn-controller connection-status 2>&1
|
|
echo "=== ovn-controller debug/status ==="
|
|
sudo env OVS_RUNDIR=/var/run/ovn ovs-appctl -t ovn-controller debug/status 2>&1 | head -20
|
|
echo "=== gw/eni Port_Binding claim detail (chassis/up/requested_chassis) ==="
|
|
sudo ovn-sbctl --no-leader-only --columns=logical_port,type,chassis,up,requested_chassis find Port_Binding 2>&1 | grep -A4 -E "gw-vpc|port-eni" | head -60
|
|
echo "=== ovn-central vs ovn-controller boot ordering ==="
|
|
systemctl show ovn-central ovn-controller openvswitch-switch -p Id -p ActiveEnterTimestamp -p ExecMainStartTimestamp 2>&1
|
|
echo "=== tail /var/log/ovn/ovn-controller.log ==="
|
|
sudo tail -n 60 /var/log/ovn/ovn-controller.log 2>&1
|
|
echo "=== tail /var/log/ovn/ovn-northd.log ==="
|
|
sudo tail -n 40 /var/log/ovn/ovn-northd.log 2>&1
|
|
`
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, bundle)
|
|
harness.DumpFile(t, fix.artifacts, "diag-ovn-claim.txt", out)
|
|
if err != nil {
|
|
harness.Detail(t, "ovn-claim", fmt.Sprintf("dump err: %v", err))
|
|
return
|
|
}
|
|
body := string(out)
|
|
switch {
|
|
case strings.Contains(body, "not connected"):
|
|
harness.Detail(t, "ovn-claim", "ovn-controller NOT connected to SB — claim failure is an SB-connection/ordering issue")
|
|
case strings.Contains(body, "chassis : []"), strings.Contains(body, "chassis : []"):
|
|
harness.Detail(t, "ovn-claim", "gw/eni Port_Binding unclaimed (chassis empty) — diff ovn-pre-chassis.txt vs this for identity churn")
|
|
default:
|
|
harness.Detail(t, "ovn-claim", "see diag-ovn-claim.txt — compare Chassis _uuid against ovn-pre-chassis.txt")
|
|
}
|
|
}
|
|
|
|
// dumpALBProbe probes the ALB public IP from host — pre-reboot this path
|
|
// worked (the burst hit it). If post-reboot ALB is also unreachable, the
|
|
// gateway/ext-bridge is broken; if ALB reaches but apps don't, only the
|
|
// app-VM side is broken. Complements per-app probes.
|
|
func dumpALBProbe(ctx context.Context, t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
if fix.albPublicIP == "" {
|
|
harness.Detail(t, "alb", "no public IP captured")
|
|
return
|
|
}
|
|
probe := fmt.Sprintf(`set +e
|
|
echo "=== ping ALB %[1]s ==="
|
|
ping -c 2 -W 2 %[1]s 2>&1
|
|
echo "=== tcp-connect ALB %[1]s:80 ==="
|
|
timeout 5 bash -c 'exec 3<>/dev/tcp/%[1]s/80 && echo "tcp OPEN" || echo "tcp CLOSED/UNREACH"'
|
|
echo "=== curl ALB http://%[1]s/ ==="
|
|
curl -sS -m 5 -w 'HTTP=%%{http_code} time=%%{time_total}s\n' http://%[1]s/ 2>&1
|
|
`, fix.albPublicIP)
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, probe)
|
|
harness.DumpFile(t, fix.artifacts, "diag-alb-probe.txt", out)
|
|
if err != nil {
|
|
harness.Detail(t, "alb", fmt.Sprintf("probe err: %v", err))
|
|
return
|
|
}
|
|
body := string(out)
|
|
switch {
|
|
case strings.Contains(body, `"instance_id"`):
|
|
harness.Detail(t, "alb", "ALB serving (responder reachable through HAProxy)")
|
|
case strings.Contains(body, "tcp OPEN"):
|
|
harness.Detail(t, "alb", "ALB tcp open, no body (HAProxy up but backends silent)")
|
|
case strings.Contains(body, "tcp CLOSED"):
|
|
harness.Detail(t, "alb", "ALB tcp closed/unreach (gateway or sys.micro broken)")
|
|
default:
|
|
harness.Detail(t, "alb", "probe inconclusive — see artifact")
|
|
}
|
|
}
|
|
|
|
// diagnoseAppInstance probes one instance from the host VM: qemu running,
|
|
// ping, TCP-connect on :80, HTTP GET. Single shell session per instance.
|
|
func diagnoseAppInstance(ctx context.Context, t *testing.T, fix *fixture, id string) {
|
|
t.Helper()
|
|
|
|
psOut, err := fix.ssh.Run(ctx, fix.env.WANHost,
|
|
fmt.Sprintf("ps -eo args= | grep -F %q | grep -v grep || true", id))
|
|
harness.DumpFile(t, fix.artifacts, fmt.Sprintf("diag-%s-qemu-ps.txt", id), psOut)
|
|
switch {
|
|
case err != nil:
|
|
harness.Detail(t, id, fmt.Sprintf("qemu ps err: %v", err))
|
|
case len(strings.TrimSpace(string(psOut))) == 0:
|
|
harness.Detail(t, id, "qemu NOT running on host")
|
|
default:
|
|
harness.Detail(t, id, "qemu running")
|
|
}
|
|
|
|
// Serial console (ttyS0) — cloud-init, kernel oops, systemd boot events.
|
|
consoleOut, _ := fix.ssh.Run(ctx, fix.env.WANHost,
|
|
fmt.Sprintf("sudo tail -c 262144 /run/spinifex/console-%s.log 2>/dev/null || true", id))
|
|
harness.DumpFile(t, fix.artifacts, fmt.Sprintf("diag-%s-console.log", id), consoleOut)
|
|
|
|
priv := fix.preIPs[id]
|
|
pub := describePublicIP(t, fix.aws, id)
|
|
harness.Detail(t, id, fmt.Sprintf("priv=%s pub=%s", priv, pub))
|
|
if pub == "" {
|
|
harness.Detail(t, id, "no public IP — cannot probe from host")
|
|
return
|
|
}
|
|
|
|
probe := fmt.Sprintf(`set +e
|
|
echo "=== ping %[1]s ==="
|
|
ping -c 2 -W 2 %[1]s 2>&1
|
|
echo "=== tcp-connect %[1]s:80 ==="
|
|
timeout 5 bash -c 'exec 3<>/dev/tcp/%[1]s/80 && echo "tcp OPEN" || echo "tcp CLOSED/UNREACH"'
|
|
echo "=== curl http://%[1]s/ ==="
|
|
curl -sS -m 5 -o /tmp/diag-body-%[2]s -w 'HTTP=%%{http_code} time=%%{time_total}s size=%%{size_download}\n' http://%[1]s/
|
|
echo "=== body ==="
|
|
cat /tmp/diag-body-%[2]s 2>/dev/null; echo
|
|
rm -f /tmp/diag-body-%[2]s
|
|
`, pub, id)
|
|
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, probe)
|
|
harness.DumpFile(t, fix.artifacts, fmt.Sprintf("diag-%s-net-probe.txt", id), out)
|
|
if err != nil {
|
|
harness.Detail(t, id, fmt.Sprintf("probe err: %v", err))
|
|
return
|
|
}
|
|
|
|
body := string(out)
|
|
switch {
|
|
case strings.Contains(body, `"instance_id"`):
|
|
harness.Detail(t, id, "responder OK from host")
|
|
case strings.Contains(body, "tcp OPEN"):
|
|
harness.Detail(t, id, "tcp open but responder silent")
|
|
case strings.Contains(body, "tcp CLOSED"):
|
|
harness.Detail(t, id, "tcp closed/unreach (responder down OR OVN path broken)")
|
|
default:
|
|
harness.Detail(t, id, "probe inconclusive — see artifact")
|
|
}
|
|
|
|
// SSH-in via IGW; dumps responder unit + cloud-init state inside the guest.
|
|
sshIntoApp(ctx, t, fix, id, pub)
|
|
}
|
|
|
|
// sshIntoApp uploads the key to the host then SSHes into the app guest to dump
|
|
// responder, cloud-init, and network state. Best-effort: no-ops if key unavailable or SSH fails.
|
|
func sshIntoApp(ctx context.Context, t *testing.T, fix *fixture, id, pub string) {
|
|
t.Helper()
|
|
if fix.privateKeyPath == "" {
|
|
return
|
|
}
|
|
keyData, err := os.ReadFile(fix.privateKeyPath)
|
|
if err != nil {
|
|
harness.Detail(t, id, fmt.Sprintf("ssh-in: read key err: %v", err))
|
|
return
|
|
}
|
|
enc := base64.StdEncoding.EncodeToString(keyData)
|
|
remoteKey := fmt.Sprintf("/tmp/reboot-e2e-%s.pem", id)
|
|
|
|
diag := `set +e
|
|
echo "=== uname / uptime ==="
|
|
uname -a; uptime
|
|
echo "=== systemctl is-active reboot-e2e-httpd.service ==="
|
|
systemctl is-active reboot-e2e-httpd.service
|
|
systemctl is-enabled reboot-e2e-httpd.service
|
|
echo "=== systemctl status (head 40) ==="
|
|
systemctl status --no-pager reboot-e2e-httpd.service 2>&1 | head -40
|
|
echo "=== unit file ==="
|
|
ls -l /etc/systemd/system/reboot-e2e-httpd.service 2>&1
|
|
echo "=== unit file content ==="
|
|
sudo cat /etc/systemd/system/reboot-e2e-httpd.service 2>&1
|
|
echo "=== enable symlink ==="
|
|
ls -l /etc/systemd/system/multi-user.target.wants/reboot-e2e-httpd.service 2>&1
|
|
echo "=== network-online.target ==="
|
|
systemctl is-active network-online.target
|
|
systemctl status --no-pager network-online.target 2>&1 | head -20
|
|
echo "=== networkctl ==="
|
|
networkctl status 2>&1 | head -40
|
|
echo "=== ip addr ==="
|
|
ip -br addr
|
|
echo "=== ip route ==="
|
|
ip route
|
|
echo "=== port :80 listeners ==="
|
|
sudo ss -tlnp 'sport = :80' 2>/dev/null
|
|
echo "=== curl localhost ==="
|
|
curl -s -m 3 -w '\nHTTP=%{http_code}\n' http://127.0.0.1/ || echo "curl localhost FAILED"
|
|
echo "=== /var/lib/httpd ==="
|
|
ls -l /var/lib/httpd 2>&1
|
|
cat /var/lib/httpd/index.html 2>&1
|
|
echo "=== responder journal ==="
|
|
sudo journalctl -u reboot-e2e-httpd.service --no-pager -n 60 2>&1
|
|
echo "=== boot list ==="
|
|
sudo journalctl --list-boots 2>&1 | tail -5
|
|
echo "=== last-boot journal grep responder ==="
|
|
sudo journalctl -b -1 --no-pager 2>&1 | grep -i "reboot-e2e\|network-online\|cloud-init" | tail -40
|
|
echo "=== this-boot journal grep responder ==="
|
|
sudo journalctl -b --no-pager 2>&1 | grep -i "reboot-e2e\|network-online\|cloud-init" | tail -40
|
|
echo "=== cloud-init result.json ==="
|
|
cat /run/cloud-init/result.json 2>&1
|
|
echo "=== cloud-init.log tail ==="
|
|
sudo tail -80 /var/log/cloud-init.log 2>&1
|
|
echo "=== cloud-init-output.log tail ==="
|
|
sudo tail -40 /var/log/cloud-init-output.log 2>&1
|
|
echo "=== instance-id sem ==="
|
|
ls -l /var/lib/cloud/instance 2>&1
|
|
ls -l /var/lib/cloud/instances/ 2>&1
|
|
echo "=== /var/lib/cloud/instances/*/sem ==="
|
|
ls -l /var/lib/cloud/instances/*/sem/ 2>&1
|
|
`
|
|
// Single outer SSH: write the key then hop into the guest. One round trip.
|
|
outerCmd := fmt.Sprintf(`set +e
|
|
umask 077 && printf '%%s' '%s' | base64 -d > %s && chmod 600 %s
|
|
ssh -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
-o ConnectTimeout=8 -o BatchMode=yes -o ServerAliveInterval=5 \
|
|
ubuntu@%s bash -s <<'EOFDIAG'
|
|
%s
|
|
EOFDIAG
|
|
rm -f %s
|
|
`, enc, remoteKey, remoteKey, remoteKey, pub, diag, remoteKey)
|
|
|
|
out, err := fix.ssh.Run(ctx, fix.env.WANHost, outerCmd)
|
|
harness.DumpFile(t, fix.artifacts, fmt.Sprintf("diag-%s-guest-ssh.txt", id), out)
|
|
if err != nil {
|
|
harness.Detail(t, id, fmt.Sprintf("ssh-in err: %v", err))
|
|
return
|
|
}
|
|
harness.Detail(t, id, "ssh-in OK (see diag-*-guest-ssh.txt)")
|
|
}
|
|
|
|
// --- Cleanup -------------------------------------------------------------
|
|
|
|
// cleanup tears down all test resources; SG is deleted last as it can't be removed
|
|
// while instance or ALB ENIs still reference it.
|
|
func cleanup(t *testing.T, fix *fixture) {
|
|
t.Helper()
|
|
harness.Phase(t, "Cleanup")
|
|
|
|
if !fix.ssh.IsReachable(context.Background(), fix.env.WANHost) {
|
|
harness.Step(t, "cleanup skipped: node unreachable")
|
|
return
|
|
}
|
|
|
|
if fix.listenerArn != "" {
|
|
_, _ = fix.aws.ELBv2.DeleteListener(&elbv2.DeleteListenerInput{ListenerArn: aws.String(fix.listenerArn)})
|
|
}
|
|
if fix.albArn != "" {
|
|
_, _ = fix.aws.ELBv2.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{LoadBalancerArn: aws.String(fix.albArn)})
|
|
}
|
|
if fix.tgArn != "" {
|
|
_, _ = fix.aws.ELBv2.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{TargetGroupArn: aws.String(fix.tgArn)})
|
|
}
|
|
|
|
if len(fix.appInstanceIDs) > 0 {
|
|
ids := make([]*string, len(fix.appInstanceIDs))
|
|
for i, id := range fix.appInstanceIDs {
|
|
ids[i] = aws.String(id)
|
|
}
|
|
_, _ = fix.aws.EC2.TerminateInstances(&ec2.TerminateInstancesInput{InstanceIds: ids})
|
|
harness.WaitForInstanceTerminated(t, fix.aws, fix.appInstanceIDs, 60*time.Second)
|
|
}
|
|
|
|
if fix.keyPairName != "" {
|
|
_, _ = fix.aws.EC2.DeleteKeyPair(&ec2.DeleteKeyPairInput{KeyName: aws.String(fix.keyPairName)})
|
|
}
|
|
|
|
// SG must come after instances + ALB ENIs are gone.
|
|
if fix.sgID != "" {
|
|
if _, err := fix.aws.EC2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
|
|
GroupId: aws.String(fix.sgID),
|
|
}); err != nil {
|
|
var aerr awserr.Error
|
|
if !errors.As(err, &aerr) || aerr.Code() != "InvalidGroup.NotFound" {
|
|
t.Logf("delete SG %s: %v", fix.sgID, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
if fix.igwID != "" && fix.vpcID != "" {
|
|
_, _ = fix.aws.EC2.DetachInternetGateway(&ec2.DetachInternetGatewayInput{
|
|
InternetGatewayId: aws.String(fix.igwID),
|
|
VpcId: aws.String(fix.vpcID),
|
|
})
|
|
_, _ = fix.aws.EC2.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{
|
|
InternetGatewayId: aws.String(fix.igwID),
|
|
})
|
|
}
|
|
if fix.subnetID != "" {
|
|
_, _ = fix.aws.EC2.DeleteSubnet(&ec2.DeleteSubnetInput{SubnetId: aws.String(fix.subnetID)})
|
|
}
|
|
if fix.vpcID != "" {
|
|
_, _ = fix.aws.EC2.DeleteVpc(&ec2.DeleteVpcInput{VpcId: aws.String(fix.vpcID)})
|
|
}
|
|
}
|
|
|
|
// --- Misc ----------------------------------------------------------------
|
|
|
|
func durationEnv(key string, def time.Duration) time.Duration {
|
|
v := os.Getenv(key)
|
|
if v == "" {
|
|
return def
|
|
}
|
|
// Accept a bare integer (treated as seconds) or a time.ParseDuration string.
|
|
if d, err := time.ParseDuration(v); err == nil {
|
|
return d
|
|
}
|
|
if d, err := time.ParseDuration(v + "s"); err == nil {
|
|
return d
|
|
}
|
|
return def
|
|
}
|