mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-21 11:46:37 +00:00
527308c342
* feat(elbv2): localize internal-LB heartbeat failures for baremetal debug Add diagnostics so a stuck-in-provisioning internal LB can be pinned to one of three points on the heartbeat path: - lb-agent failure log now carries the gateway target + region, so the microVM serial console shows whether the heartbeat hit a transport error (never reached AWSGW) or an HTTP-level rejection. - AWSGW LBAgentHeartbeat logs receipt (debug) and warns on the previously-silent not-found/account-mismatch reject path, so the journal distinguishes 'heartbeat arrived but cannot transition' from 'heartbeat never arrived'. * test(e2e/lb): fail fast on stuck internal LB + probe fw_cfg on microvm boot - lb suite: gate the remaining internal subtests on Internal_ALB. The internal LBs share one mgmt-return datapath and fixture, so if the first never reaches active the rest time out identically — skip them instead of burning ~5m per subtest (~20m saved on a known-bad run). - microvm init.sh: widen the fw_cfg netcfg poll window and, on the not-available path, dump DMI / loaded modules / qemu_fw_cfg by_name listing / dmesg fw_cfg to the serial console before init exits, so a boot panic is diagnosable from the captured console log. * fix(microvm): insmod fallback for fw_cfg + fail build on bad modules.dep Baremetal-built microVM images shipped an unusable modules.dep (the build host's depmod errors were swallowed), so busybox modprobe in the guest init silently no-oped, qemu_fw_cfg never loaded, /sys/firmware/qemu_fw_cfg was absent, init could not read opt/spinifex/netcfg, and the LB microVM panicked before lb-agent ran — leaving internal LBs stuck in provisioning. init.sh: load_mod() tries modprobe, then falls back to insmod by explicit path (.ko/.ko.gz) which needs no modules.dep, so the guest boots regardless of which host built the image. build-microvm-image.sh: require depmod, drop the error swallow, and assert modules.dep is non-empty after depmod so a boot-broken image fails the build loudly instead of silently. * fix(microvm): decompress kernel modules before depmod Debian trixie's kmod 34 depmod silently emits an empty modules.dep when fed Alpine's gzip-compressed (.ko.gz) modules — reproduced: 928 modules in, 0-byte modules.dep out. The guest's busybox modprobe then no-ops, qemu_fw_cfg never loads, and internal LB microVMs panic before lb-agent runs (stuck provisioning on bare metal; GitHub-hosted runners ship a kmod that reads .gz, so normal CI was unaffected). Decompress kept modules to plain .ko after the keep-list strip, before depmod. Host depmod then produces a usable map regardless of its gzip support, and the guest loads modules without a decompressor. * feat(e2e): capacity-aware scheduling for dense single-node runs Denominate daemon schedulable capacity in physical cores instead of logical threads, and auto-serialise e2e launches when the cluster is at capacity, so a single hyperthreaded node stops over-packing guests. Daemon: NewResourceManager sizes hostVCPU from physicalCoreCount() (distinct physical-id:core-id pairs in /proc/cpuinfo) rather than runtime.NumCPU(). Falls back to logical CPUs when topology is absent (non-Linux, container, ARM cpuinfo without core id). On a hyperthreaded box this halves the over-pack: with the unchanged 2 vCPU reserve the i5-12600H schedules 10 vCPU (12 cores - 2) instead of 14, keeping the boot-spike CPU within real cores. Harness: capacityRetryEC2 decorates the EC2 client and, on InsufficientInstanceCapacity, waits and retries RunInstances instead of failing — turning the daemon admission gate into back-pressure so concurrent subtests queue. AWSClient.EC2 is now the wrapped ec2iface.EC2API; EC2Conf keeps the raw *ec2.EC2 for .Config access. Adds unit tests for parsePhysicalCores (hyperthreaded/hybrid/dual-socket/ fallback) and the retry decorator (retry-then-succeed, non-capacity passthrough, deadline). * fix(daemon): SPINIFEX_HOST_VCPU override so small CI hosts can start Denominating capacity by physical cores starves tiny hosts: a 2-physical- core CI runner leaves 0 schedulable after the default 2-vCPU reserve, so NewResourceManager refuses to start and every daemon-constructing unit test fails. The previous runtime.NumCPU() count (4 logical on the same runner) incidentally cleared the reserve floor. Add a SPINIFEX_HOST_VCPU escape hatch (symmetric with the existing SPINIFEX_RESERVED_* overrides) that pins the detected core count for hosts whose /proc/cpuinfo topology misreports, and pin it to 4 in the daemon TestMain so unit tests are deterministic regardless of runner topology. Reserve stays at the default 2.
124 lines
3.6 KiB
Go
124 lines
3.6 KiB
Go
//go:build e2e
|
|
|
|
package harness
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/tls"
|
|
"encoding/xml"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
|
|
)
|
|
|
|
// PostAWSAction sends a raw application/x-www-form-urlencoded POST with
|
|
// Action=<action> + Version=2016-11-15 + caller params, signed with the same
|
|
// credentials the AWS SDK is using. Gateway rejects on auth before parsing
|
|
// the action, so signing is mandatory even for InvalidAction probes.
|
|
//
|
|
// Returns the HTTP status, raw body, and the parsed <Code> from the AWS XML
|
|
// error envelope (empty for 2xx). t.Fatal on transport / signing failure.
|
|
func PostAWSAction(t *testing.T, env *Env, c *AWSClient, action string, params map[string]string) (status int, body []byte, awsCode string) {
|
|
t.Helper()
|
|
|
|
endpoint := c.EC2Conf.Endpoint
|
|
if endpoint == "" {
|
|
t.Fatalf("PostAWSAction: EC2 endpoint not configured")
|
|
}
|
|
region := aws.StringValue(c.EC2Conf.Config.Region)
|
|
|
|
form := url.Values{}
|
|
form.Set("Action", action)
|
|
if _, ok := params["Version"]; !ok {
|
|
// EC2 query API version — same value the SDK injects on every call.
|
|
// Hard-coded here because we are bypassing the SDK entirely.
|
|
form.Set("Version", "2016-11-15")
|
|
}
|
|
for k, v := range params {
|
|
form.Set(k, v)
|
|
}
|
|
bodyBytes := []byte(form.Encode())
|
|
|
|
req, err := http.NewRequest(http.MethodPost, strings.TrimRight(endpoint, "/")+"/", bytes.NewReader(bodyBytes))
|
|
if err != nil {
|
|
t.Fatalf("PostAWSAction: build request: %v", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
|
|
|
|
creds := c.EC2Conf.Config.Credentials
|
|
if creds == nil {
|
|
t.Fatalf("PostAWSAction: no credentials on client")
|
|
}
|
|
signer := v4.NewSigner(creds)
|
|
// SignedReader keeps the body intact for replay; v4.Sign reads-then-resets.
|
|
if _, err := signer.Sign(req, bytes.NewReader(bodyBytes), "ec2", region, time.Now()); err != nil {
|
|
t.Fatalf("PostAWSAction: sign: %v", err)
|
|
}
|
|
|
|
// Reuse the EC2 client's HTTP transport so TLS trust matches the rest of
|
|
// the suite. Fall back to a CA-loaded transport if the embedded client
|
|
// is using the SDK default.
|
|
httpClient := c.EC2Conf.Config.HTTPClient
|
|
if httpClient == nil {
|
|
httpClient = &http.Client{
|
|
Transport: &http.Transport{TLSClientConfig: &tls.Config{}},
|
|
Timeout: env.DefaultTimeout,
|
|
}
|
|
}
|
|
resp, err := httpClient.Do(req)
|
|
if err != nil {
|
|
t.Fatalf("PostAWSAction: %v", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
body, err = io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
t.Fatalf("PostAWSAction: read body: %v", err)
|
|
}
|
|
status = resp.StatusCode
|
|
|
|
if status >= 200 && status < 300 {
|
|
return status, body, ""
|
|
}
|
|
awsCode = parseAWSXMLErrorCode(body)
|
|
return status, body, awsCode
|
|
}
|
|
|
|
// EC2 returns either <Response><Errors><Error><Code>…</Code> or the SDK-style
|
|
// <ErrorResponse><Error><Code>…</Code> envelope. Try both shapes; the
|
|
// gateway has shipped one then the other across releases.
|
|
type ec2ErrorResponse struct {
|
|
XMLName xml.Name `xml:"Response"`
|
|
Errors struct {
|
|
Error struct {
|
|
Code string `xml:"Code"`
|
|
Message string `xml:"Message"`
|
|
} `xml:"Error"`
|
|
} `xml:"Errors"`
|
|
}
|
|
|
|
type sdkErrorResponse struct {
|
|
XMLName xml.Name `xml:"ErrorResponse"`
|
|
Error struct {
|
|
Code string `xml:"Code"`
|
|
Message string `xml:"Message"`
|
|
} `xml:"Error"`
|
|
}
|
|
|
|
func parseAWSXMLErrorCode(body []byte) string {
|
|
var e1 ec2ErrorResponse
|
|
if err := xml.Unmarshal(body, &e1); err == nil && e1.Errors.Error.Code != "" {
|
|
return e1.Errors.Error.Code
|
|
}
|
|
var e2 sdkErrorResponse
|
|
if err := xml.Unmarshal(body, &e2); err == nil && e2.Error.Code != "" {
|
|
return e2.Error.Code
|
|
}
|
|
return ""
|
|
}
|