mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
527308c342
* feat(elbv2): localize internal-LB heartbeat failures for baremetal debug Add diagnostics so a stuck-in-provisioning internal LB can be pinned to one of three points on the heartbeat path: - lb-agent failure log now carries the gateway target + region, so the microVM serial console shows whether the heartbeat hit a transport error (never reached AWSGW) or an HTTP-level rejection. - AWSGW LBAgentHeartbeat logs receipt (debug) and warns on the previously-silent not-found/account-mismatch reject path, so the journal distinguishes 'heartbeat arrived but cannot transition' from 'heartbeat never arrived'. * test(e2e/lb): fail fast on stuck internal LB + probe fw_cfg on microvm boot - lb suite: gate the remaining internal subtests on Internal_ALB. The internal LBs share one mgmt-return datapath and fixture, so if the first never reaches active the rest time out identically — skip them instead of burning ~5m per subtest (~20m saved on a known-bad run). - microvm init.sh: widen the fw_cfg netcfg poll window and, on the not-available path, dump DMI / loaded modules / qemu_fw_cfg by_name listing / dmesg fw_cfg to the serial console before init exits, so a boot panic is diagnosable from the captured console log. * fix(microvm): insmod fallback for fw_cfg + fail build on bad modules.dep Baremetal-built microVM images shipped an unusable modules.dep (the build host's depmod errors were swallowed), so busybox modprobe in the guest init silently no-oped, qemu_fw_cfg never loaded, /sys/firmware/qemu_fw_cfg was absent, init could not read opt/spinifex/netcfg, and the LB microVM panicked before lb-agent ran — leaving internal LBs stuck in provisioning. init.sh: load_mod() tries modprobe, then falls back to insmod by explicit path (.ko/.ko.gz) which needs no modules.dep, so the guest boots regardless of which host built the image. build-microvm-image.sh: require depmod, drop the error swallow, and assert modules.dep is non-empty after depmod so a boot-broken image fails the build loudly instead of silently. * fix(microvm): decompress kernel modules before depmod Debian trixie's kmod 34 depmod silently emits an empty modules.dep when fed Alpine's gzip-compressed (.ko.gz) modules — reproduced: 928 modules in, 0-byte modules.dep out. The guest's busybox modprobe then no-ops, qemu_fw_cfg never loads, and internal LB microVMs panic before lb-agent runs (stuck provisioning on bare metal; GitHub-hosted runners ship a kmod that reads .gz, so normal CI was unaffected). Decompress kept modules to plain .ko after the keep-list strip, before depmod. Host depmod then produces a usable map regardless of its gzip support, and the guest loads modules without a decompressor. * feat(e2e): capacity-aware scheduling for dense single-node runs Denominate daemon schedulable capacity in physical cores instead of logical threads, and auto-serialise e2e launches when the cluster is at capacity, so a single hyperthreaded node stops over-packing guests. Daemon: NewResourceManager sizes hostVCPU from physicalCoreCount() (distinct physical-id:core-id pairs in /proc/cpuinfo) rather than runtime.NumCPU(). Falls back to logical CPUs when topology is absent (non-Linux, container, ARM cpuinfo without core id). On a hyperthreaded box this halves the over-pack: with the unchanged 2 vCPU reserve the i5-12600H schedules 10 vCPU (12 cores - 2) instead of 14, keeping the boot-spike CPU within real cores. Harness: capacityRetryEC2 decorates the EC2 client and, on InsufficientInstanceCapacity, waits and retries RunInstances instead of failing — turning the daemon admission gate into back-pressure so concurrent subtests queue. AWSClient.EC2 is now the wrapped ec2iface.EC2API; EC2Conf keeps the raw *ec2.EC2 for .Config access. Adds unit tests for parsePhysicalCores (hyperthreaded/hybrid/dual-socket/ fallback) and the retry decorator (retry-then-succeed, non-capacity passthrough, deadline). * fix(daemon): SPINIFEX_HOST_VCPU override so small CI hosts can start Denominating capacity by physical cores starves tiny hosts: a 2-physical- core CI runner leaves 0 schedulable after the default 2-vCPU reserve, so NewResourceManager refuses to start and every daemon-constructing unit test fails. The previous runtime.NumCPU() count (4 logical on the same runner) incidentally cleared the reserve floor. Add a SPINIFEX_HOST_VCPU escape hatch (symmetric with the existing SPINIFEX_RESERVED_* overrides) that pins the detected core count for hosts whose /proc/cpuinfo topology misreports, and pin it to 4 in the daemon TestMain so unit tests are deterministic regardless of runner topology. Reserve stays at the default 2.
179 lines
7.1 KiB
Go
179 lines
7.1 KiB
Go
//go:build e2e
|
||
|
||
package single
|
||
|
||
import (
|
||
"fmt"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/aws/aws-sdk-go/aws"
|
||
"github.com/aws/aws-sdk-go/service/ec2"
|
||
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
||
"github.com/stretchr/testify/assert"
|
||
"github.com/stretchr/testify/require"
|
||
)
|
||
|
||
// runSnapshotLifecycle creates, describes, copies, and deletes a
|
||
// snapshot off the running instance's root volume (the only volume that's
|
||
// definitely backed by a mounted viperblock instance — create-snapshot
|
||
// requires that). Maps to run-e2e.sh ~627–786.
|
||
func runSnapshotLifecycle(t *testing.T, fix *Fixture) {
|
||
harness.Phase(t, "Single — Snapshot Lifecycle")
|
||
|
||
_, rootVolumeID := needInstance(t, fix)
|
||
|
||
// Pin the API-reported root volume size so we can cross-check the
|
||
// snapshot's VolumeSize field.
|
||
descVol, err := fix.AWS.EC2.DescribeVolumes(&ec2.DescribeVolumesInput{
|
||
VolumeIds: []*string{aws.String(rootVolumeID)},
|
||
})
|
||
require.NoError(t, err, "describe-volumes %s", rootVolumeID)
|
||
require.NotEmpty(t, descVol.Volumes, "no volume for %s", rootVolumeID)
|
||
rootSize := aws.Int64Value(descVol.Volumes[0].Size)
|
||
|
||
// Local CreateSnapshot — this phase deletes the snapshot in-line, so do
|
||
// NOT route through harness.EnsureSnapshot (its fixture cleanup would
|
||
// later try to re-delete).
|
||
harness.Step(t, "create-snapshot volume=%s", rootVolumeID)
|
||
const origDesc = "e2e-test-snapshot"
|
||
createOut, err := fix.AWS.EC2.CreateSnapshot(&ec2.CreateSnapshotInput{
|
||
VolumeId: aws.String(rootVolumeID),
|
||
Description: aws.String(origDesc),
|
||
})
|
||
require.NoError(t, err, "create-snapshot")
|
||
snapshotID := aws.StringValue(createOut.SnapshotId)
|
||
require.NotEmpty(t, snapshotID, "CreateSnapshot returned empty SnapshotId")
|
||
t.Cleanup(func() {
|
||
_, _ = fix.AWS.EC2.DeleteSnapshot(&ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapshotID)})
|
||
})
|
||
harness.WaitForSnapshotState(t, fix.AWS, snapshotID, "completed", harness.WithPoll(500*time.Millisecond))
|
||
harness.Detail(t, "snapshot", snapshotID, "size", rootSize)
|
||
|
||
harness.Step(t, "describe-snapshots %s", snapshotID)
|
||
desc, err := fix.AWS.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
|
||
SnapshotIds: []*string{aws.String(snapshotID)},
|
||
})
|
||
require.NoError(t, err, "describe-snapshots %s", snapshotID)
|
||
require.NotEmpty(t, desc.Snapshots, "describe-snapshots returned nothing")
|
||
got := desc.Snapshots[0]
|
||
assert.Equal(t, rootVolumeID, aws.StringValue(got.VolumeId), "describe VolumeId mismatch")
|
||
assert.Equal(t, rootSize, aws.Int64Value(got.VolumeSize), "describe VolumeSize mismatch")
|
||
assert.Equal(t, origDesc, aws.StringValue(got.Description), "describe Description mismatch")
|
||
|
||
// CopySnapshot expects the source region — pulled off the configured
|
||
// EC2 client so the test honours SPINIFEX_AWS_REGION overrides.
|
||
region := aws.StringValue(fix.AWS.EC2Conf.Config.Region)
|
||
const copyDesc = "e2e-copy"
|
||
harness.Step(t, "copy-snapshot src=%s region=%s", snapshotID, region)
|
||
copyOut, err := fix.AWS.EC2.CopySnapshot(&ec2.CopySnapshotInput{
|
||
SourceSnapshotId: aws.String(snapshotID),
|
||
SourceRegion: aws.String(region),
|
||
Description: aws.String(copyDesc),
|
||
})
|
||
require.NoError(t, err, "copy-snapshot")
|
||
copySnapshotID := aws.StringValue(copyOut.SnapshotId)
|
||
require.NotEmpty(t, copySnapshotID, "copy-snapshot returned empty SnapshotId")
|
||
require.NotEqual(t, snapshotID, copySnapshotID, "copy snapshot ID should differ from original")
|
||
t.Cleanup(func() {
|
||
_, _ = fix.AWS.EC2.DeleteSnapshot(&ec2.DeleteSnapshotInput{SnapshotId: aws.String(copySnapshotID)})
|
||
})
|
||
harness.Detail(t, "copy_snapshot", copySnapshotID)
|
||
|
||
harness.WaitForSnapshotState(t, fix.AWS, copySnapshotID, "completed", harness.WithPoll(500*time.Millisecond))
|
||
|
||
harness.Step(t, "describe-snapshots %s,%s (both visible)", snapshotID, copySnapshotID)
|
||
both, err := fix.AWS.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
|
||
SnapshotIds: []*string{aws.String(snapshotID), aws.String(copySnapshotID)},
|
||
})
|
||
require.NoError(t, err, "describe-snapshots both")
|
||
require.Lenf(t, both.Snapshots, 2, "expected 2 snapshots, got %d", len(both.Snapshots))
|
||
|
||
// Verify the copy carries the new description.
|
||
var copyDescGot string
|
||
for _, s := range both.Snapshots {
|
||
if aws.StringValue(s.SnapshotId) == copySnapshotID {
|
||
copyDescGot = aws.StringValue(s.Description)
|
||
break
|
||
}
|
||
}
|
||
assert.Equal(t, copyDesc, copyDescGot, "copy Description mismatch")
|
||
|
||
harness.Step(t, "delete-snapshot %s (original)", snapshotID)
|
||
_, err = fix.AWS.EC2.DeleteSnapshot(&ec2.DeleteSnapshotInput{
|
||
SnapshotId: aws.String(snapshotID),
|
||
})
|
||
require.NoError(t, err, "delete-snapshot original")
|
||
|
||
harness.EventuallyErr(t, func() error {
|
||
out, err := fix.AWS.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
|
||
SnapshotIds: []*string{aws.String(snapshotID)},
|
||
})
|
||
if err != nil {
|
||
if harness.ErrorCodeIs(err, "InvalidSnapshot.NotFound") {
|
||
return nil
|
||
}
|
||
return fmt.Errorf("describe-snapshots: %w", err)
|
||
}
|
||
if len(out.Snapshots) == 0 {
|
||
return nil
|
||
}
|
||
return fmt.Errorf("snapshot %s still visible (state=%s)",
|
||
snapshotID, aws.StringValue(out.Snapshots[0].State))
|
||
}, 2*time.Minute, 2*time.Second)
|
||
|
||
harness.Step(t, "verify copy %s still completed", copySnapshotID)
|
||
copyDescOut, err := fix.AWS.EC2.DescribeSnapshots(&ec2.DescribeSnapshotsInput{
|
||
SnapshotIds: []*string{aws.String(copySnapshotID)},
|
||
})
|
||
require.NoError(t, err, "describe-snapshots copy")
|
||
require.NotEmpty(t, copyDescOut.Snapshots, "copy disappeared after original delete")
|
||
assert.Equal(t, "completed", aws.StringValue(copyDescOut.Snapshots[0].State),
|
||
"copy snapshot should remain completed after original delete")
|
||
|
||
harness.Step(t, "delete-snapshot %s (copy)", copySnapshotID)
|
||
_, err = fix.AWS.EC2.DeleteSnapshot(&ec2.DeleteSnapshotInput{
|
||
SnapshotId: aws.String(copySnapshotID),
|
||
})
|
||
require.NoError(t, err, "delete-snapshot copy")
|
||
|
||
harness.Detail(t, "deleted_original", snapshotID)
|
||
}
|
||
|
||
// runSnapshotBackedLaunch verifies the AMI used for the live instance
|
||
// carries a snapshot reference — proof the launch went through the
|
||
// cloneAMIToVolume → OpenFromSnapshot path. Maps to run-e2e.sh ~788–803.
|
||
//
|
||
// Bash's prose mentions verifying the predastore-side snapshot config, but
|
||
// the actual bash only checks the EC2 API. We follow the bash.
|
||
func runSnapshotBackedLaunch(t *testing.T, fix *Fixture) {
|
||
harness.Phase(t, "Single — Snapshot-Backed Instance Launch")
|
||
|
||
amiID := needAMI(t, fix)
|
||
|
||
out, err := fix.AWS.EC2.DescribeImages(&ec2.DescribeImagesInput{
|
||
ImageIds: []*string{aws.String(amiID)},
|
||
})
|
||
require.NoError(t, err, "describe-images %s", amiID)
|
||
require.NotEmpty(t, out.Images, "no image for %s", amiID)
|
||
|
||
var snapID string
|
||
for _, bdm := range out.Images[0].BlockDeviceMappings {
|
||
if bdm.Ebs == nil {
|
||
continue
|
||
}
|
||
if id := aws.StringValue(bdm.Ebs.SnapshotId); id != "" {
|
||
snapID = id
|
||
break
|
||
}
|
||
}
|
||
require.NotEmptyf(t, snapID,
|
||
"AMI %s has no BlockDeviceMappings[].Ebs.SnapshotId — launch was NOT snapshot-backed", amiID)
|
||
harness.Detail(t, "ami", amiID, "snapshot", snapID)
|
||
|
||
// TODO(stage-?): bash mentions verifying the predastore-side
|
||
// `snap-{amiId}/config.json` exists with SnapshotID + SourceVolumeName
|
||
// populated. The current bash doesn't actually do that — if we ever
|
||
// want to enforce it we need an S3 client wired through the harness.
|
||
}
|