feat: cloud-init EC2 datasource (#411)

2026-10-22 12:16:52 +00:00 · 2026-06-25 13:48:08 +10:00
parent 5e58645192
commit 8a94886b13
81 changed files with 1050 additions and 1496 deletions
@@ -15,7 +15,7 @@
 #
 # Optional:
 #   SSH_KEY        (default: ~/.ssh/spinifex-key)
-#   SSH_USER       (default: ec2-user)
+#   SSH_USER       (default: ubuntu)

 set -euo pipefail

@@ -26,7 +26,7 @@ QWEN_IP="${QWEN_IP:?set QWEN_IP}"
 VIDEO_PATH="${VIDEO_PATH:?set VIDEO_PATH}"

 SSH_KEY="${SSH_KEY:-$HOME/.ssh/spinifex-key}"
-SSH_USER="${SSH_USER:-ec2-user}"
+SSH_USER="${SSH_USER:-ubuntu}"

 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

@@ -40,6 +40,8 @@ scripts/images/eks-node/mulga-ebs-byid.initd:/etc/init.d/mulga-ebs-byid \
 scripts/images/eks-node/mulga-ebs-byid.sh:/usr/local/sbin/mulga-ebs-byid \
 scripts/images/eks-node/mulga-eks-provider-id.initd:/etc/init.d/mulga-eks-provider-id \
 scripts/images/eks-node/mulga-eks-provider-id.sh:/usr/local/sbin/mulga-eks-provider-id \
+scripts/images/eks-node/mulga-mgmt-net.initd:/etc/init.d/mulga-mgmt-net \
+scripts/images/eks-node/mulga-mgmt-net.sh:/usr/local/sbin/mulga-mgmt-net \
 scripts/images/eks-node/addons/spinifex-noop/0.1.0/noop.yaml:/usr/share/spinifex-eks/addons/spinifex-noop/0.1.0/noop.yaml \
 scripts/images/eks-node/addons/argocd/3.0.23/argocd.yaml:/usr/share/spinifex-eks/addons/argocd/3.0.23/argocd.yaml \
 scripts/images/eks-node/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml:/usr/share/spinifex-eks/addons/aws-load-balancer-controller/2.11.0/00-crds.yaml \
@@ -52,10 +54,14 @@ scripts/images/eks-node/addons/aws-ebs-csi-driver/1.40.1/30-storageclass.yaml:/u
 scripts/images/eks-node/coredns/coredns-mulga.yaml:/usr/share/spinifex-eks/coredns/coredns-mulga.yaml \
 scripts/images/eks-node/mulga-eks-etcd-snapshot.sh:/etc/periodic/daily/mulga-eks-etcd-snapshot"

-# Only crond (etcd snapshot) + the role selector at bake. The selector adds the
-# role-specific services (k3s/eks-token-webhook/k3s-first-boot/state-report or
-# k3s-agent) to the default runlevel on first boot.
-ENABLE_SERVICES="crond eks-node-role mulga-ebs-byid mulga-eks-provider-id"
+# crond (etcd snapshot), the role selector, and the role-agnostic node services
+# (EBS by-id, provider-id, mgmt NIC) at bake. mulga-mgmt-net configures the
+# system-VM NICs from fw_cfg before cloud-init/k3s — DHCP on the primary data
+# NIC so the Ec2 datasource reaches IMDS, static on mgmt0 (no-op without a
+# netcfg blob). The selector adds the role-specific services
+# (k3s/eks-token-webhook/k3s-first-boot/state-report or k3s-agent) to the
+# default runlevel on first boot.
+ENABLE_SERVICES="crond eks-node-role mulga-ebs-byid mulga-eks-provider-id mulga-mgmt-net"

 SETUP_SCRIPT="scripts/images/eks-node/setup.sh"

@@ -0,0 +1,19 @@
+#!/sbin/openrc-run
+# mulga-mgmt-net — boot oneshot that configures a multi-NIC system VM's NICs from
+# the fw_cfg netcfg blob before cloud-init's network stage. BootAMI system VMs
+# (EKS control plane) are dual-NIC: the data ENI is DHCP'd here so the Ec2 IMDS
+# datasource can reach 169.254.169.254 (cloud-init can't pick the right NIC out
+# of two on Alpine), and mgmt0's static br-mgmt address is applied. Role-agnostic
+# and enabled on every node — a no-op when no netcfg blob is present.
+
+description="Configure system-VM NICs from fw_cfg netcfg (data DHCP + mgmt static)"
+
+depend() {
+    before cloud-init k3s k3s-agent
+}
+
+start() {
+    ebegin "Configuring system-VM NICs from fw_cfg netcfg"
+    /usr/local/sbin/mulga-mgmt-net
+    eend $?
+}
@@ -0,0 +1,95 @@
+#!/bin/sh
+# mulga-mgmt-net — boot oneshot that brings a multi-NIC BootAMI system VM's NICs
+# up from the QEMU fw_cfg netcfg blob (opt/spinifex/netcfg) the host writes per
+# launch. The EKS control plane is the case: cloud-init on a stock Alpine guest
+# cannot reliably pick the right NIC out of two, so the host enumerates them and
+# this oneshot configures each by MAC, before cloud-init's network stage:
+#   - NIC<n>_DHCP=1: the primary data ENI. OVN serves DHCP; we lease it (one-shot)
+#     so the Ec2 IMDS datasource can reach 169.254.169.254, and pin a /32 to it so
+#     a link-local 169.254.0.0/16 route on another NIC cannot hijack IMDS.
+#   - NIC<n>_CIDR: a static NIC (mgmt0 on br-mgmt, which has no DHCP). Applied
+#     address-only; NIC<n>_DEFAULT is 0, so mgmt0 is never the default route.
+#
+# The blob is shell KEY=value, matching daemon.buildNetcfgBlob and
+# build/microvm/init.sh; interfaces are matched by MAC. No-op when the blob is
+# absent (a single-NIC agent/worker brings its one NIC up via cloud-init/IMDS),
+# so the same image boots unchanged without one.
+set -eu
+
+NETCFG="${MULGA_NETCFG:-/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw}"
+IMDS_IP="169.254.169.254"
+
+# qemu_fw_cfg is a module on the stock Alpine cloud image; load it (best-effort)
+# before reading. Skipped harmlessly when built-in or already loaded.
+modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true
+
+if [ ! -f "$NETCFG" ]; then
+    echo "[mulga-mgmt-net] no fw_cfg netcfg; skipping system NIC setup"
+    exit 0
+fi
+
+# shellcheck disable=SC1090
+. "$NETCFG"
+
+# One-shot DHCP: configure the link then exit, leaving no daemon to fight
+# cloud-init's later management of the primary ENI. Prefer busybox udhcpc; fall
+# back to dhcpcd where udhcpc's config script is absent.
+dhcp_oneshot() {
+    iface="$1"
+    if command -v udhcpc >/dev/null 2>&1 && [ -x /usr/share/udhcpc/default.script ]; then
+        udhcpc -i "$iface" -f -q -n -t 8 -T 2 >/dev/null 2>&1
+    elif command -v dhcpcd >/dev/null 2>&1; then
+        dhcpcd -q -t 20 "$iface" >/dev/null 2>&1
+    else
+        echo "[mulga-mgmt-net] no DHCP client (udhcpc/dhcpcd) for $iface" >&2
+        return 1
+    fi
+}
+
+for n in 0 1 2 3 4 5; do
+    # ${VAR:-} keeps the unset NIC<n>_* slots safe under `set -u`.
+    eval "mac=\${NIC${n}_MAC:-}"
+    [ -z "$mac" ] && continue
+
+    iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
+    if [ -z "$iface" ]; then
+        echo "[mulga-mgmt-net] WARNING: no interface for MAC $mac (NIC$n); skipping" >&2
+        continue
+    fi
+
+    eval "dhcp=\${NIC${n}_DHCP:-}"
+    eval "cidr=\${NIC${n}_CIDR:-}"
+    ip link set "$iface" up
+
+    if [ "$dhcp" = "1" ]; then
+        if dhcp_oneshot "$iface"; then
+            echo "[mulga-mgmt-net] data NIC $iface ($mac) up via DHCP"
+        else
+            echo "[mulga-mgmt-net] ERROR: no DHCP lease on data NIC $iface ($mac)" >&2
+        fi
+        # Pin IMDS to the data NIC so a link-local 169.254.0.0/16 route on another
+        # interface cannot steal the metadata path. Route via the gateway, not
+        # on-link: the host demuxes IMDS sent to the gateway MAC, never ARP-answers .254.
+        gw=$(ip -4 route show default dev "$iface" 2>/dev/null | awk '{print $3; exit}')
+        if [ -n "$gw" ]; then
+            ip route replace "${IMDS_IP}/32" via "$gw" dev "$iface" 2>/dev/null || true
+        else
+            ip route replace "${IMDS_IP}/32" dev "$iface" 2>/dev/null || true
+        fi
+        continue
+    fi
+
+    if [ -z "$cidr" ]; then
+        echo "[mulga-mgmt-net] brought up $iface ($mac), no CIDR"
+        continue
+    fi
+    # `replace` is idempotent — adds the address if absent, no-op on a stop/start
+    # re-attach that reuses a surviving interface. A real failure must surface,
+    # not strand mgmt0 IP-less.
+    if ip addr replace "$cidr" dev "$iface"; then
+        echo "[mulga-mgmt-net] configured $iface ($mac) with $cidr"
+    else
+        echo "[mulga-mgmt-net] ERROR: failed to set $cidr on $iface ($mac)" >&2
+        exit 1
+    fi
+done
@@ -49,6 +49,7 @@ chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync
 chmod 0755 /etc/init.d/mulga-ebs-byid /usr/local/sbin/mulga-ebs-byid
 chmod 0755 /etc/init.d/mulga-eks-provider-id /usr/local/sbin/mulga-eks-provider-id
+chmod 0755 /etc/init.d/mulga-mgmt-net /usr/local/sbin/mulga-mgmt-net
 chmod 0755 /etc/periodic/daily/mulga-eks-etcd-snapshot

 # EBS by-id bridge: route every virtio-blk event through mulga-ebs-byid, which
@@ -6,7 +6,7 @@ export AWS_PROFILE=spinifex
 # Resolve the actual user's home — $HOME is /root when run under sudo.
 _USER_HOME=$(getent passwd "${SUDO_USER:-$(id -un)}" | cut -d: -f6)
 SSH_KEY="${SSH_KEY:-${_USER_HOME}/.ssh/spinifex-key}"
-SSH_USER="${SSH_USER:-ec2-user}"
+SSH_USER="${SSH_USER:-ubuntu}"
 SSH_TIMEOUT="${SSH_TIMEOUT:-300}"

 usage() {
@@ -28,7 +28,7 @@ Options:

 Env overrides:
  SSH_KEY      Path to SSH private key (default: ~/.ssh/spinifex-key)
-  SSH_USER     SSH user inside guest  (default: ec2-user)
+  SSH_USER     SSH user inside guest  (default: ubuntu)
  SSH_TIMEOUT  Seconds to wait for initial SSH (default: 300)
 EOF
 }
@@ -137,7 +137,7 @@ SSH_READY=false

 while [ $SSH_ATTEMPTS -lt $MAX_SSH_ATTEMPTS ]; do
    ssh-keyscan -p "$SSH_PORT" 127.0.0.1 >> ~/.ssh/known_hosts 2>/dev/null
-    if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p "$SSH_PORT" -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 'echo ready' >/dev/null 2>&1; then
+    if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=2 -p "$SSH_PORT" -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 'echo ready' >/dev/null 2>&1; then
        SSH_READY=true
        echo "SSH is ready!"
        break
@@ -156,7 +156,7 @@ if [ "$SSH_READY" != "true" ]; then
 fi

 echo "Copying benchmark script to instance..."
-scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$SSH_PORT" -i ~/.ssh/spinifex-key "$BENCH_SCRIPT" ec2-user@127.0.0.1:~/disk-performance.sh
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$SSH_PORT" -i ~/.ssh/spinifex-key "$BENCH_SCRIPT" ubuntu@127.0.0.1:~/disk-performance.sh

 if [ $? -ne 0 ]; then
    echo "Error: Failed to copy benchmark script to instance"
@@ -165,7 +165,7 @@ fi

 # Execute benchmark script
 echo "Running benchmark on instance..."
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p "$SSH_PORT" -i ~/.ssh/spinifex-key ec2-user@127.0.0.1 'chmod +x ~/disk-performance.sh && ~/disk-performance.sh' > /tmp/spinifex-vm-disk.log 2>&1
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p "$SSH_PORT" -i ~/.ssh/spinifex-key ubuntu@127.0.0.1 'chmod +x ~/disk-performance.sh && ~/disk-performance.sh' > /tmp/spinifex-vm-disk.log 2>&1

 if [ $? -ne 0 ]; then
    echo "Error: Benchmark execution failed, check /tmp/spinifex-vm-disk.log for details"
@@ -226,7 +226,7 @@ for _i in $(seq 1 300); do
           -o BatchMode=yes \
           -p "$SSH_INST_PORT" \
           -i "$SSH_KEY" \
-           ec2-user@"$SSH_INST_HOST" 'echo ready' >/dev/null 2>&1; then
+           ubuntu@"$SSH_INST_HOST" 'echo ready' >/dev/null 2>&1; then
        SSH_READY=1
        break
    fi
@@ -246,10 +246,10 @@ SSH_OUT=$(ssh -o StrictHostKeyChecking=no \
    -o BatchMode=yes \
    -p "$SSH_INST_PORT" \
    -i "$SSH_KEY" \
-    ec2-user@"$SSH_INST_HOST" 'id && hostname' 2>&1)
+    ubuntu@"$SSH_INST_HOST" 'id && hostname' 2>&1)
 echo "  $SSH_OUT"
-if ! echo "$SSH_OUT" | grep -q "ec2-user"; then
-    echo "❌ Expected ec2-user in SSH output"
+if ! echo "$SSH_OUT" | grep -q "ubuntu"; then
+    echo "❌ Expected ubuntu in SSH output"
    exit 1
 fi

@@ -261,7 +261,7 @@ if [[ "$TEST_GPU" == "1" ]]; then
        if ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
               -o ConnectTimeout=5 -o BatchMode=yes \
               -p "$SSH_INST_PORT" -i "$SSH_KEY" \
-               ec2-user@"$SSH_INST_HOST" \
+               ubuntu@"$SSH_INST_HOST" \
               'lspci 2>/dev/null | grep -iE "nvidia|amdgpu|instinct"' 2>/dev/null; then
            GPU_VISIBLE=1
            break