mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
Josh Siviour
527308c342
* feat(elbv2): localize internal-LB heartbeat failures for baremetal debug Add diagnostics so a stuck-in-provisioning internal LB can be pinned to one of three points on the heartbeat path: - lb-agent failure log now carries the gateway target + region, so the microVM serial console shows whether the heartbeat hit a transport error (never reached AWSGW) or an HTTP-level rejection. - AWSGW LBAgentHeartbeat logs receipt (debug) and warns on the previously-silent not-found/account-mismatch reject path, so the journal distinguishes 'heartbeat arrived but cannot transition' from 'heartbeat never arrived'. * test(e2e/lb): fail fast on stuck internal LB + probe fw_cfg on microvm boot - lb suite: gate the remaining internal subtests on Internal_ALB. The internal LBs share one mgmt-return datapath and fixture, so if the first never reaches active the rest time out identically — skip them instead of burning ~5m per subtest (~20m saved on a known-bad run). - microvm init.sh: widen the fw_cfg netcfg poll window and, on the not-available path, dump DMI / loaded modules / qemu_fw_cfg by_name listing / dmesg fw_cfg to the serial console before init exits, so a boot panic is diagnosable from the captured console log. * fix(microvm): insmod fallback for fw_cfg + fail build on bad modules.dep Baremetal-built microVM images shipped an unusable modules.dep (the build host's depmod errors were swallowed), so busybox modprobe in the guest init silently no-oped, qemu_fw_cfg never loaded, /sys/firmware/qemu_fw_cfg was absent, init could not read opt/spinifex/netcfg, and the LB microVM panicked before lb-agent ran — leaving internal LBs stuck in provisioning. init.sh: load_mod() tries modprobe, then falls back to insmod by explicit path (.ko/.ko.gz) which needs no modules.dep, so the guest boots regardless of which host built the image. build-microvm-image.sh: require depmod, drop the error swallow, and assert modules.dep is non-empty after depmod so a boot-broken image fails the build loudly instead of silently. * fix(microvm): decompress kernel modules before depmod Debian trixie's kmod 34 depmod silently emits an empty modules.dep when fed Alpine's gzip-compressed (.ko.gz) modules — reproduced: 928 modules in, 0-byte modules.dep out. The guest's busybox modprobe then no-ops, qemu_fw_cfg never loads, and internal LB microVMs panic before lb-agent runs (stuck provisioning on bare metal; GitHub-hosted runners ship a kmod that reads .gz, so normal CI was unaffected). Decompress kept modules to plain .ko after the keep-list strip, before depmod. Host depmod then produces a usable map regardless of its gzip support, and the guest loads modules without a decompressor. * feat(e2e): capacity-aware scheduling for dense single-node runs Denominate daemon schedulable capacity in physical cores instead of logical threads, and auto-serialise e2e launches when the cluster is at capacity, so a single hyperthreaded node stops over-packing guests. Daemon: NewResourceManager sizes hostVCPU from physicalCoreCount() (distinct physical-id:core-id pairs in /proc/cpuinfo) rather than runtime.NumCPU(). Falls back to logical CPUs when topology is absent (non-Linux, container, ARM cpuinfo without core id). On a hyperthreaded box this halves the over-pack: with the unchanged 2 vCPU reserve the i5-12600H schedules 10 vCPU (12 cores - 2) instead of 14, keeping the boot-spike CPU within real cores. Harness: capacityRetryEC2 decorates the EC2 client and, on InsufficientInstanceCapacity, waits and retries RunInstances instead of failing — turning the daemon admission gate into back-pressure so concurrent subtests queue. AWSClient.EC2 is now the wrapped ec2iface.EC2API; EC2Conf keeps the raw *ec2.EC2 for .Config access. Adds unit tests for parsePhysicalCores (hyperthreaded/hybrid/dual-socket/ fallback) and the retry decorator (retry-then-succeed, non-capacity passthrough, deadline). * fix(daemon): SPINIFEX_HOST_VCPU override so small CI hosts can start Denominating capacity by physical cores starves tiny hosts: a 2-physical- core CI runner leaves 0 schedulable after the default 2-vCPU reserve, so NewResourceManager refuses to start and every daemon-constructing unit test fails. The previous runtime.NumCPU() count (4 logical on the same runner) incidentally cleared the reserve floor. Add a SPINIFEX_HOST_VCPU escape hatch (symmetric with the existing SPINIFEX_RESERVED_* overrides) that pins the detected core count for hosts whose /proc/cpuinfo topology misreports, and pin it to 4 in the daemon TestMain so unit tests are deterministic regardless of runner topology. Reserve stays at the default 2.
92 lines
2.7 KiB
Go
92 lines
2.7 KiB
Go
//go:build e2e
|
|
|
|
package harness
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
|
"github.com/aws/aws-sdk-go/service/ec2"
|
|
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
|
|
)
|
|
|
|
// fakeCapacityEC2 returns capErr for the first failCount RunInstances calls, then a
|
|
// non-nil reservation. Tracks how many times RunInstances was invoked.
|
|
type fakeCapacityEC2 struct {
|
|
ec2iface.EC2API
|
|
failCount int
|
|
calls int
|
|
capErr error
|
|
}
|
|
|
|
func (f *fakeCapacityEC2) RunInstances(*ec2.RunInstancesInput) (*ec2.Reservation, error) {
|
|
f.calls++
|
|
if f.calls <= f.failCount {
|
|
return nil, f.capErr
|
|
}
|
|
return &ec2.Reservation{}, nil
|
|
}
|
|
|
|
func capacityErr() error {
|
|
return awserr.New("InsufficientInstanceCapacity", "cluster full", nil)
|
|
}
|
|
|
|
func TestCapacityRetry_RetriesThenSucceeds(t *testing.T) {
|
|
fake := &fakeCapacityEC2{failCount: 2, capErr: capacityErr()}
|
|
c := &capacityRetryEC2{EC2API: fake, interval: time.Millisecond, maxWait: time.Minute}
|
|
|
|
out, err := c.RunInstances(&ec2.RunInstancesInput{})
|
|
if err != nil {
|
|
t.Fatalf("expected success after retries, got %v", err)
|
|
}
|
|
if out == nil {
|
|
t.Fatalf("expected non-nil reservation")
|
|
}
|
|
if fake.calls != 3 {
|
|
t.Fatalf("expected 3 calls (2 fail + 1 ok), got %d", fake.calls)
|
|
}
|
|
}
|
|
|
|
func TestCapacityRetry_NonCapacityErrorReturnsImmediately(t *testing.T) {
|
|
boom := errors.New("AuthFailure: bad creds")
|
|
fake := &fakeCapacityEC2{failCount: 1, capErr: boom}
|
|
c := &capacityRetryEC2{EC2API: fake, interval: time.Millisecond, maxWait: time.Minute}
|
|
|
|
_, err := c.RunInstances(&ec2.RunInstancesInput{})
|
|
if !errors.Is(err, boom) {
|
|
t.Fatalf("expected the non-capacity error back, got %v", err)
|
|
}
|
|
if fake.calls != 1 {
|
|
t.Fatalf("non-capacity error must not retry, got %d calls", fake.calls)
|
|
}
|
|
}
|
|
|
|
func TestCapacityRetry_DeadlineReturnsLastError(t *testing.T) {
|
|
// Always at capacity; a tiny maxWait forces the loop to give up and surface
|
|
// the capacity error rather than hang.
|
|
fake := &fakeCapacityEC2{failCount: 1 << 30, capErr: capacityErr()}
|
|
c := &capacityRetryEC2{EC2API: fake, interval: time.Millisecond, maxWait: 5 * time.Millisecond}
|
|
|
|
_, err := c.RunInstances(&ec2.RunInstancesInput{})
|
|
if !isInsufficientCapacity(err) {
|
|
t.Fatalf("expected capacity error after deadline, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestIsInsufficientCapacity(t *testing.T) {
|
|
if !isInsufficientCapacity(capacityErr()) {
|
|
t.Fatalf("awserr InsufficientInstanceCapacity should match")
|
|
}
|
|
if !isInsufficientCapacity(errors.New("got InsufficientInstanceCapacity from gw")) {
|
|
t.Fatalf("string-wrapped capacity error should match")
|
|
}
|
|
if isInsufficientCapacity(nil) {
|
|
t.Fatalf("nil must not match")
|
|
}
|
|
if isInsufficientCapacity(errors.New("AuthFailure")) {
|
|
t.Fatalf("unrelated error must not match")
|
|
}
|
|
}
|