mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-20 03:11:46 +00:00
527308c342
* feat(elbv2): localize internal-LB heartbeat failures for baremetal debug Add diagnostics so a stuck-in-provisioning internal LB can be pinned to one of three points on the heartbeat path: - lb-agent failure log now carries the gateway target + region, so the microVM serial console shows whether the heartbeat hit a transport error (never reached AWSGW) or an HTTP-level rejection. - AWSGW LBAgentHeartbeat logs receipt (debug) and warns on the previously-silent not-found/account-mismatch reject path, so the journal distinguishes 'heartbeat arrived but cannot transition' from 'heartbeat never arrived'. * test(e2e/lb): fail fast on stuck internal LB + probe fw_cfg on microvm boot - lb suite: gate the remaining internal subtests on Internal_ALB. The internal LBs share one mgmt-return datapath and fixture, so if the first never reaches active the rest time out identically — skip them instead of burning ~5m per subtest (~20m saved on a known-bad run). - microvm init.sh: widen the fw_cfg netcfg poll window and, on the not-available path, dump DMI / loaded modules / qemu_fw_cfg by_name listing / dmesg fw_cfg to the serial console before init exits, so a boot panic is diagnosable from the captured console log. * fix(microvm): insmod fallback for fw_cfg + fail build on bad modules.dep Baremetal-built microVM images shipped an unusable modules.dep (the build host's depmod errors were swallowed), so busybox modprobe in the guest init silently no-oped, qemu_fw_cfg never loaded, /sys/firmware/qemu_fw_cfg was absent, init could not read opt/spinifex/netcfg, and the LB microVM panicked before lb-agent ran — leaving internal LBs stuck in provisioning. init.sh: load_mod() tries modprobe, then falls back to insmod by explicit path (.ko/.ko.gz) which needs no modules.dep, so the guest boots regardless of which host built the image. build-microvm-image.sh: require depmod, drop the error swallow, and assert modules.dep is non-empty after depmod so a boot-broken image fails the build loudly instead of silently. * fix(microvm): decompress kernel modules before depmod Debian trixie's kmod 34 depmod silently emits an empty modules.dep when fed Alpine's gzip-compressed (.ko.gz) modules — reproduced: 928 modules in, 0-byte modules.dep out. The guest's busybox modprobe then no-ops, qemu_fw_cfg never loads, and internal LB microVMs panic before lb-agent runs (stuck provisioning on bare metal; GitHub-hosted runners ship a kmod that reads .gz, so normal CI was unaffected). Decompress kept modules to plain .ko after the keep-list strip, before depmod. Host depmod then produces a usable map regardless of its gzip support, and the guest loads modules without a decompressor. * feat(e2e): capacity-aware scheduling for dense single-node runs Denominate daemon schedulable capacity in physical cores instead of logical threads, and auto-serialise e2e launches when the cluster is at capacity, so a single hyperthreaded node stops over-packing guests. Daemon: NewResourceManager sizes hostVCPU from physicalCoreCount() (distinct physical-id:core-id pairs in /proc/cpuinfo) rather than runtime.NumCPU(). Falls back to logical CPUs when topology is absent (non-Linux, container, ARM cpuinfo without core id). On a hyperthreaded box this halves the over-pack: with the unchanged 2 vCPU reserve the i5-12600H schedules 10 vCPU (12 cores - 2) instead of 14, keeping the boot-spike CPU within real cores. Harness: capacityRetryEC2 decorates the EC2 client and, on InsufficientInstanceCapacity, waits and retries RunInstances instead of failing — turning the daemon admission gate into back-pressure so concurrent subtests queue. AWSClient.EC2 is now the wrapped ec2iface.EC2API; EC2Conf keeps the raw *ec2.EC2 for .Config access. Adds unit tests for parsePhysicalCores (hyperthreaded/hybrid/dual-socket/ fallback) and the retry decorator (retry-then-succeed, non-capacity passthrough, deadline). * fix(daemon): SPINIFEX_HOST_VCPU override so small CI hosts can start Denominating capacity by physical cores starves tiny hosts: a 2-physical- core CI runner leaves 0 schedulable after the default 2-vCPU reserve, so NewResourceManager refuses to start and every daemon-constructing unit test fails. The previous runtime.NumCPU() count (4 logical on the same runner) incidentally cleared the reserve floor. Add a SPINIFEX_HOST_VCPU escape hatch (symmetric with the existing SPINIFEX_RESERVED_* overrides) that pins the detected core count for hosts whose /proc/cpuinfo topology misreports, and pin it to 4 in the daemon TestMain so unit tests are deterministic regardless of runner topology. Reserve stays at the default 2.
87 lines
3.3 KiB
Bash
Executable File
87 lines
3.3 KiB
Bash
Executable File
#!/bin/sh
|
|
mount -t proc proc /proc
|
|
mount -t sysfs sys /sys
|
|
mount -t devtmpfs dev /dev
|
|
|
|
# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
|
|
# already enumerated by the time PID 1 runs — no modprobe needed.
|
|
# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
|
|
#
|
|
# modprobe needs a usable modules.dep; when the image is built on a host whose
|
|
# depmod can't produce one, busybox modprobe silently no-ops and the module
|
|
# never loads — fw_cfg then never appears and init panics. insmod by explicit
|
|
# path needs no modules.dep, so fall back to it. Handles .ko and .ko.gz.
|
|
load_mod() {
|
|
modprobe "$1" 2>/dev/null && return 0
|
|
for ko in $(find /lib/modules -name "$1.ko" -o -name "$1.ko.gz" 2>/dev/null); do
|
|
insmod "$ko" 2>/dev/null && return 0
|
|
case "$ko" in
|
|
*.gz) gunzip -c "$ko" >/tmp/mod.ko 2>/dev/null \
|
|
&& insmod /tmp/mod.ko 2>/dev/null && return 0 ;;
|
|
esac
|
|
done
|
|
return 1
|
|
}
|
|
load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
|
|
load_mod virtio_net || true
|
|
|
|
NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
|
|
i=0
|
|
while [ "$i" -lt 25 ]; do
|
|
[ -f "$NETCFG" ] && break
|
|
sleep 0.04
|
|
i=$((i + 1))
|
|
done
|
|
|
|
if [ ! -f "$NETCFG" ]; then
|
|
# Dump host/guest fw_cfg state to the serial console before init exits so a
|
|
# "netcfg not available" panic is diagnosable from the captured console log
|
|
# (module-not-loaded vs qemu-not-exposing-by_name vs wrong machine type).
|
|
echo "[init] FATAL: fw_cfg netcfg not available" >&2
|
|
echo "[init] dmi: vendor=$(cat /sys/class/dmi/id/sys_vendor 2>/dev/null) product=$(cat /sys/class/dmi/id/product_name 2>/dev/null) bios=$(cat /sys/class/dmi/id/bios_version 2>/dev/null)" >&2
|
|
echo "[init] qemu_fw_cfg dir present: $([ -d /sys/firmware/qemu_fw_cfg ] && echo yes || echo NO)" >&2
|
|
echo "[init] modules: $(lsmod 2>/dev/null | awk 'NR>1{print $1}' | tr '\n' ',')" >&2
|
|
echo "[init] by_name listing:" >&2
|
|
ls -la /sys/firmware/qemu_fw_cfg/by_name/ 2>&1 | sed 's/^/[init] /' >&2
|
|
ls -la /sys/firmware/qemu_fw_cfg/by_name/opt/ 2>&1 | sed 's/^/[init] /' >&2
|
|
echo "[init] dmesg fw_cfg:" >&2
|
|
dmesg 2>/dev/null | grep -i fw_cfg | sed 's/^/[init] /' >&2
|
|
exit 1
|
|
fi
|
|
|
|
. "$NETCFG"
|
|
|
|
for n in 0 1 2 3 4 5; do
|
|
eval mac=\$NIC${n}_MAC
|
|
[ -z "$mac" ] && continue
|
|
|
|
iface=""
|
|
for attempt in 1 2 3 4 5; do
|
|
iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
|
|
[ -n "$iface" ] && break
|
|
sleep 0.02
|
|
done
|
|
|
|
if [ -z "$iface" ]; then
|
|
echo "[init] WARNING: no interface found for MAC $mac (NIC$n), skipping" >&2
|
|
continue
|
|
fi
|
|
|
|
eval cidr=\$NIC${n}_CIDR
|
|
eval gw=\$NIC${n}_GW
|
|
eval default=\$NIC${n}_DEFAULT
|
|
eval rdst=\$NIC${n}_ROUTE_DST
|
|
eval rvia=\$NIC${n}_ROUTE_VIA
|
|
|
|
ip link set "$iface" up
|
|
[ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
|
|
[ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
|
|
[ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
|
|
done
|
|
|
|
cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/lb-agent-env/raw /etc/conf.d/lb-agent
|
|
mkdir -p /etc/ssl/certs
|
|
cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw /etc/ssl/certs/ca-certificates.crt
|
|
|
|
exec /sbin/init
|