gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: enable nightly nginx-alb workbook (#161)

Browse Source
This commit is contained in:
Julian Sommer
2026-04-24 17:14:57 +10:00
committed by GitHub
parent 5126a1df8d
commit eeb7ebc5db
8 changed files with 211 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e-nightly.yml
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
include:
- { cell: 2, install: binary, topo: single, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-single, extra: "" }
- { cell: 9, install: binary, topo: real-multi, net: static, host: ubuntu-24.04, arch: x86_64, runner: ci-multi, extra: "" }
- { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-single, extra: "tofu-examples" }
- { cell: 17, install: binary, topo: single, net: static, host: debian-13, arch: x86_64, runner: ci-pseudo, extra: "tofu-examples" }
runs-on: [ self-hosted, "${{ matrix.runner }}" ]
timeout-minutes: 35
spinifex/daemon/daemon.go
+24 -11
View File
@@ -2783,11 +2783,21 @@ func (d *Daemon) wireLBAgentConfig() {
// Resolve gateway URL — the address LB VMs use to reach the AWS gateway.
// Precedence:
// 1. AWSGW listens on a specific WAN IP + br-mgmt present → use WAN IP
// and add host route via br-mgmt for internal LBs.
// 2. br-mgmt present + AWSGW on 0.0.0.0 → br-mgmt IP (both LB flavours
// 1. br-mgmt present + AWSGW on a dedicated IP distinct from AdvertiseIP
// (multi-node: AWSGW on a mgmt-only IP, VPC path can't reach it) →
// gateway URL is the AWSGW bind IP and lb-agent gets a bootcmd host
// route via br-mgmt.
// 2. AdvertiseIP set (single-node, or multi-node where AWSGW binds to
// the advertised IP) → AdvertiseIP. VMs reach it via VPC → external
// (OVN's own dnat_and_snat SNATs their reply back to the ALB EIP).
// Critically, we do NOT add the mgmt host route here: when host IPs
// on the WAN share the advertiseIP, the /32 route would steal the
// return path for host-initiated ALB connections — replies would
// egress via mgmt with the VM's 10.x source IP, bypass OVN's SNAT,
// and arrive at the host with a source that doesn't match the open
// TCP socket (the client dialed the EIP, not the VM IP).
// 3. br-mgmt present + AWSGW on 0.0.0.0 → br-mgmt IP (both LB flavours
// reach the daemon via mgmt).
// 3. This node's AdvertiseIP (single-node default install) → AdvertiseIP.
// 4. DevNetworking shim → 10.0.2.2.
// 5. AWSGW bound to specific IP (no br-mgmt, no advertise) → that IP.
// 6. Else: error and skip assignment — no silent empty URL.
@@ -2802,17 +2812,20 @@ func (d *Daemon) wireLBAgentConfig() {
advertiseIP := d.config.AdvertiseIP
switch {
case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" && !net.ParseIP(awsgwBindIP).IsLoopback():
// Multi-node: AWSGW listens on a specific WAN IP. Use that IP as the
// gateway URL; internal LBs get a host route via br-mgmt.
case d.mgmtBridgeIP != "" && awsgwBindIP != "" && awsgwBindIP != "0.0.0.0" &&
!net.ParseIP(awsgwBindIP).IsLoopback() && awsgwBindIP != advertiseIP:
// Multi-node: AWSGW on a dedicated mgmt IP. VMs can't reach it via
// VPC → external, so add a bootcmd host route via br-mgmt.
gatewayHost = awsgwBindIP
d.mgmtRouteVia = awsgwBindIP
case d.mgmtBridgeIP != "":
// Multi-node with AWSGW on 0.0.0.0 — br-mgmt reaches all LBs.
gatewayHost = d.mgmtBridgeIP
case advertiseIP != "" && advertiseIP != "0.0.0.0":
// Single-node default install — off-host clients dial the advertised IP.
// Single-node, or multi-node where AWSGW binds to AdvertiseIP: VMs
// reach AWSGW via the normal VPC → external path. No mgmt host route.
gatewayHost = advertiseIP
case d.mgmtBridgeIP != "":
// br-mgmt present + AWSGW on 0.0.0.0 and no advertiseIP — br-mgmt IP
// is the only reachable address.
gatewayHost = d.mgmtBridgeIP
case d.config.Daemon.DevNetworking:
gatewayHost = "10.0.2.2"
case awsgwBindIP != "" && awsgwBindIP != "0.0.0.0":
spinifex/daemon/daemon_wire_lb_test.go
+41
View File
@@ -202,3 +202,44 @@ func TestWireLBAgentConfig_GatewayURL_NoAdvertiseNoBridge(t *testing.T) {
assert.Empty(t, d.elbv2Service.GatewayURL)
}
// Single-node local dev: AWSGW on 0.0.0.0, br-mgmt present, AdvertiseIP set.
// Gateway must be AdvertiseIP (reachable via the VM's WAN NIC), NOT the
// br-mgmt IP (guest has no DHCP on br-mgmt so that subnet is unreachable).
func TestWireLBAgentConfig_GatewayURL_AdvertiseOverMgmtBridge(t *testing.T) {
cfg := &config.Config{
AdvertiseIP: "192.168.1.32",
AWSGW: config.AWSGWConfig{Host: "0.0.0.0:9999"},
}
d := newWireLBTestDaemon(t, cfg)
d.mgmtBridgeIP = "10.14.8.1"
d.wireLBAgentConfig()
assert.Equal(t, "https://192.168.1.32:9999", d.elbv2Service.GatewayURL)
assert.Empty(t, d.mgmtRouteVia)
assert.Empty(t, d.elbv2Service.MgmtRouteGateway)
assert.Empty(t, d.elbv2Service.MgmtRouteTarget)
}
// Single-node with br-mgmt AND a specific AWSGW bind that matches AdvertiseIP
// (e.g. `spx admin init --bind $PRIMARY_WAN`): the WAN IP is reachable via the
// normal VPC → external path, so the bootcmd mgmt host route must NOT be
// injected. Injecting it would divert the reply leg of host-initiated ALB
// connections out br-mgmt, bypassing OVN SNAT and arriving at the host with a
// source that doesn't match the open TCP socket.
func TestWireLBAgentConfig_GatewayURL_SingleNodeBindMatchesAdvertise(t *testing.T) {
cfg := &config.Config{
AdvertiseIP: "192.168.0.17",
AWSGW: config.AWSGWConfig{Host: "192.168.0.17:9999"},
}
d := newWireLBTestDaemon(t, cfg)
d.mgmtBridgeIP = "10.15.8.1"
d.wireLBAgentConfig()
assert.Equal(t, "https://192.168.0.17:9999", d.elbv2Service.GatewayURL)
assert.Empty(t, d.mgmtRouteVia)
assert.Empty(t, d.elbv2Service.MgmtRouteGateway)
assert.Empty(t, d.elbv2Service.MgmtRouteTarget)
}
spinifex/handlers/ec2/eip/service_impl.go
+6 -1
View File
@@ -569,12 +569,17 @@ func (s *EIPServiceImpl) findByAssociationID(accountID, associationID string) (*
// publishNATEvent publishes a NAT lifecycle event to NATS for vpcd consumption.
// This is fire-and-forget; errors are logged but do not fail the API response.
//
// PortName must match the OVN logical switch port name ("port-<eni-id>") because
// vpcd sets NAT.LogicalPort to this value in distributed NAT mode (direct
// bridge). A mismatch creates a dnat_and_snat row pointing at a nonexistent
// port, and OVN never programs the DNAT flow.
func (s *EIPServiceImpl) publishNATEvent(topic, vpcID, externalIP, logicalIP, eniID, mac string) {
utils.PublishEvent(s.natsConn, topic, natEvent{
VpcId: vpcID,
ExternalIP: externalIP,
LogicalIP: logicalIP,
PortName: eniID,
PortName: "port-" + eniID,
MAC: mac,
})
}
spinifex/handlers/ec2/eip/service_impl_test.go
+30
View File
@@ -3,6 +3,7 @@ package handlers_ec2_eip
import (
"encoding/json"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
@@ -555,3 +556,32 @@ func TestEIP_DescribeAddresses_FilterByTag(t *testing.T) {
assert.Len(t, desc.Addresses, 1)
assert.Equal(t, *out.AllocationId, *desc.Addresses[0].AllocationId)
}
// TestEIP_PublishNATEvent_PortNameHasPortPrefix is a regression test for a bug
// where EIPServiceImpl.publishNATEvent sent the raw ENI id as PortName. vpcd
// writes PortName into NAT.LogicalPort in distributed NAT mode (direct bridge),
// so a mismatch with the OVN logical switch port name ("port-<eni>") produces
// a dnat_and_snat row pointing at a nonexistent port — OVN never programs the
// flow and the EIP black-holes. See mulga-js-17.
func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) {
svc, _ := setupTestEIP(t)
sub, err := svc.natsConn.SubscribeSync("vpc.add-nat")
require.NoError(t, err)
defer func() { _ = sub.Unsubscribe() }()
const eniID = "eni-abc123"
svc.publishNATEvent("vpc.add-nat", "vpc-1", "198.51.100.10", "10.0.0.5", eniID, "02:00:00:00:00:01")
msg, err := sub.NextMsg(2 * time.Second)
require.NoError(t, err)
var got natEvent
require.NoError(t, json.Unmarshal(msg.Data, &got))
assert.Equal(t, "port-"+eniID, got.PortName,
"PortName must match the OVN logical switch port name (port-<eni>); "+
"otherwise vpcd's distributed NAT creates a dnat_and_snat rule "+
"pointing at a nonexistent logical port and the flow is never programmed")
assert.Equal(t, "10.0.0.5", got.LogicalIP)
assert.Equal(t, "198.51.100.10", got.ExternalIP)
}
tests/e2e/run-e2e.sh
+3 -3
View File
@@ -388,7 +388,7 @@ else
fi
# Wait for SSH to become ready (VM boot + cloud-init)
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60
# Test basic SSH connectivity
test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "test-key-1.pem"
@@ -1044,7 +1044,7 @@ fi
echo "SSH endpoint: $SSH_HOST:$SSH_PORT"
# Wait for SSH to become ready
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60
# Verify vCPU count matches the new instance type (nproc reports online CPUs)
echo "Verifying vCPU count inside the VM..."
@@ -1119,7 +1119,7 @@ else
SSH_PORT=$(get_ssh_port "$INSTANCE_ID")
SSH_HOST=$(get_ssh_host "$INSTANCE_ID")
fi
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 30
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "test-key-1.pem" 60
# Verify guest actually rebooted (uptime < 120 seconds)
UPTIME_SECS=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
tests/e2e/run-pseudo-multinode-e2e.sh
+1 -1
View File
@@ -385,7 +385,7 @@ for idx in "${!INSTANCE_IDS[@]}"; do
SSH_HOSTS+=("$SSH_HOST")
# Wait for SSH to become ready (VM boot + cloud-init)
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 30
wait_for_ssh "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem" 60
# Test basic SSH connectivity
test_ssh_connectivity "$SSH_HOST" "$SSH_PORT" "multinode-test-key.pem"
tests/e2e/run-tofu-examples-e2e.sh
+105 -32
View File
@@ -5,6 +5,9 @@
# brought up a single-node cluster. For each workbook, the driver does a
# clean tofu init/apply, runs one smoke assertion, then tofu destroy.
#
# All assertions are minimal by design (one check per workbook) so the
# maintenance cost of keeping them in sync with workbook edits stays low.
#
# Expects the workbooks to be available at $WORKBOOK_DIR (defaults to the
# tarball-included ./workbooks/ directory beside this script).
@@ -16,12 +19,14 @@ export AWS_PROFILE=spinifex
WORKBOOK_DIR="${WORKBOOK_DIR:-${SCRIPT_DIR}/workbooks}"
OPENTOFU_VERSION="${OPENTOFU_VERSION:-1.11.5}"
# awsgw and predastore bind to the WAN IP (bootstrap-install passes --bind
# ${PRIMARY_WAN}), so workbooks need the WAN IP for both the tofu provider
# endpoints and any CLI assertions that talk to S3 (:8443).
# awsgw and predastore are bound to the WAN IP (bootstrap-install passes --bind
# ${PRIMARY_WAN}), not loopback. Workbooks need the WAN IP for both the tofu
# provider endpoints and any CLI assertions that talk to S3 (:8443).
WAN_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}')
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR)
CURRENT_WORKBOOK=""
log() { echo "[$(date +%H:%M:%S)] $*"; }
install_tofu() {
@@ -41,6 +46,37 @@ install_tofu() {
rm -rf "$tmp"
}
cleanup() {
EXIT_CODE=$?
if [ "$EXIT_CODE" -ne 0 ]; then
log "=== FAIL (workbook=${CURRENT_WORKBOOK:-<none>}) — dumping diagnostics ==="
log "--- spx get vms ---"
sudo -u spinifex-daemon spx get vms 2>&1 | head -80 || \
spx get vms 2>&1 | head -80 || true
log "--- aws ec2 describe-instances ---"
aws ec2 describe-instances \
--query 'Reservations[].Instances[].[InstanceId,State.Name,StateReason.Message,PrivateIpAddress,PublicIpAddress,Tags[?Key==`Name`].Value|[0]]' \
--output table 2>&1 | head -60 || true
log "--- spinifex service errors (last 15 min) ---"
for svc in spinifex-daemon spinifex-awsgw spinifex-vpcd spinifex-viperblock spinifex-predastore; do
sudo journalctl -u "$svc" --since '15 min ago' --no-pager 2>/dev/null | \
grep -iE 'panic|fatal|level=error|ERROR' | tail -30 | \
sed "s|^| [${svc}] |" || true
done
log "--- service journals (last 200 lines each) ---"
for svc in spinifex-nats spinifex-predastore spinifex-viperblock \
spinifex-daemon spinifex-awsgw spinifex-vpcd; do
echo "=== ${svc} ==="
sudo journalctl -u "${svc}" --no-pager -n 200 2>/dev/null || true
done
fi
exit "$EXIT_CODE"
}
trap cleanup EXIT
# --- Per-workbook assertions ---
# Wait up to 150s for instance SSH readiness.
wait_for_ssh() {
local key="$1" host="$2"
for _ in $(seq 1 30); do
@@ -52,11 +88,14 @@ wait_for_ssh() {
return 1
}
# Wait up to $2 seconds (default 150) for a 200 response from $1.
wait_for_http_200() {
local url="$1" budget="${2:-150}"
local attempts=$((budget / 5))
for _ in $(seq 1 "$attempts"); do
if [ "$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null)" = "200" ]; then
local status
status=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null || echo "000")
if [ "$status" = "200" ]; then
return 0
fi
sleep 5
@@ -64,6 +103,7 @@ wait_for_http_200() {
return 1
}
# Wait up to $2 seconds for the target group $1 to report at least one healthy target.
wait_for_alb_healthy() {
local tg_arn="$1" budget="${2:-300}"
local attempts=$((budget / 10))
@@ -87,46 +127,76 @@ assert_bastion_private_subnet() {
key="$(pwd)/bastion-demo.pem"
chmod 600 "$key"
wait_for_ssh "$key" "$bastion" || return 1
# sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem,
# so wait for the hop key before attempting the inner ssh.
ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \
'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1' || return 1
wait_for_ssh "$key" "$bastion" || {
log " bastion SSH not ready"
return 1
}
# sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem.
# Wait up to 180s for user_data to drop the key before hopping.
if ! ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \
'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1'; then
log " bastion: ~/.ssh/bastion-demo.pem never appeared (cloud-init stalled?)"
return 1
fi
ssh "${SSH_OPTS[@]}" -i "$key" "ec2-user@${bastion}" \
"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ~/.ssh/bastion-demo.pem ec2-user@${private} id" \
| grep -q '^uid='
}
assert_nginx_webserver() {
wait_for_http_200 "http://$(tofu output -raw public_ip)/"
}
assert_nginx_alb() {
# ALB DNS (*.elb.spinifex.local) isn't resolvable from the host — README
# documents fetching the public IP via elbv2 describe-load-balancers.
local name ip tg_arn
local name ip tg_arn attempt
name=$(tofu output -raw alb_name)
ip=$(aws elbv2 describe-load-balancers --names "$name" \
--query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
--output text 2>/dev/null | awk '{print $1}')
# Retry describe-load-balancers: the gateway intermittently returns
# InternalError during ALB provisioning while the lb-agent VM is booting.
ip=""
for attempt in 1 2 3 4 5 6; do
ip=$(aws elbv2 describe-load-balancers --names "$name" \
--query 'LoadBalancers[0].AvailabilityZones[].LoadBalancerAddresses[].IpAddress' \
--output text 2>/dev/null | awk '{print $1}')
if [ -n "$ip" ] && [ "$ip" != "None" ]; then
break
fi
log " nginx-alb: describe-load-balancers attempt ${attempt} returned no IP, retrying in 10s"
sleep 10
done
if [ -z "$ip" ] || [ "$ip" = "None" ]; then
log " nginx-alb: no public IP for ${name}"
log " nginx-alb: no public IP for ${name} after retries"
return 1
fi
log " nginx-alb: ALB ${name} public IP ${ip}"
tg_arn=$(aws elbv2 describe-target-groups --names nginx-alb-tg \
--query 'TargetGroups[0].TargetGroupArn' --output text 2>/dev/null)
if [ -z "$tg_arn" ] || [ "$tg_arn" = "None" ]; then
log " nginx-alb: could not resolve target group ARN"
return 1
fi
wait_for_alb_healthy "$tg_arn" 300 || return 1
wait_for_http_200 "http://${ip}/" 300
wait_for_alb_healthy "$tg_arn" 300 || {
log " nginx-alb: no healthy targets after 300s"
return 1
}
wait_for_http_200 "http://${ip}/" 300 || {
log " nginx-alb: targets healthy but no HTTP 200 after 300s"
return 1
}
}
assert_nginx_webserver() {
local ip
ip=$(tofu output -raw public_ip)
wait_for_http_200 "http://${ip}/"
}
assert_s3_webapp() {
# aws s3 goes to predastore on :8443, not awsgw on :9999 — the workbook's
# provider sets s3 = predastore_endpoint but the CLI doesn't inherit that.
local bucket endpoint sentinel
# aws s3 goes to predastore on :8443, not the awsgw on :9999 the default
# profile points at — the workbook's provider sets s3 = predastore_endpoint
# but the CLI doesn't inherit that.
local bucket sentinel endpoint
bucket=$(tofu output -raw bucket_name)
endpoint="https://${WAN_IP}:8443"
sentinel="spinifex-nightly-$(date +%s).txt"
@@ -135,8 +205,9 @@ assert_s3_webapp() {
}
# Pick an instance type available on this cluster. Workbooks default to
# t3.small (Intel); on AMD-only hosts t3 isn't registered, so query and
# fall back to the smallest type with ≥2 vCPU / ≥1 GiB RAM.
# t3.small (Intel); on AMD-only hosts t3 isn't registered, so we query and
# fall back to the smallest type with ≥2 vCPU / ≥1 GiB RAM (matches the
# nginx-alb README's documented approach).
detect_instance_type() {
local endpoint="https://${WAN_IP}:9999"
local picked
@@ -152,6 +223,8 @@ detect_instance_type() {
INSTANCE_TYPE=""
# --- Workbook driver ---
run_workbook() {
local example="$1"
local path="${WORKBOOK_DIR}/${example}"
@@ -161,6 +234,7 @@ run_workbook() {
return 1
fi
CURRENT_WORKBOOK="$example"
log "=== ${example} ==="
cd "$path"
rm -rf .terraform terraform.tfstate* .terraform.lock.hcl
@@ -171,7 +245,8 @@ run_workbook() {
"-var=instance_type=${INSTANCE_TYPE}"
)
# s3-webapp has extra required vars; creds come from AWS_PROFILE=spinifex.
# s3-webapp has three required-no-default vars; creds come from
# AWS_PROFILE=spinifex so the workbook's boto3 client authenticates.
if [ "$example" = "s3-webapp" ]; then
local access_key secret_key
access_key=$(aws configure get aws_access_key_id --profile spinifex)
@@ -210,6 +285,8 @@ run_workbook() {
return "$rc"
}
# --- Main ---
install_tofu || { log "tofu install failed"; exit 1; }
INSTANCE_TYPE=$(detect_instance_type)
@@ -219,16 +296,12 @@ if [ -z "$INSTANCE_TYPE" ] || [ "$INSTANCE_TYPE" = "None" ]; then
fi
log "Using instance_type=${INSTANCE_TYPE}"
# nginx-alb is disabled pending: (1) vpcd port-group live OVN client
# (CreatePortGroup/AddACL not yet implemented → SG rules are no-ops),
# (2) viperblock zero-write block-alignment bug that crashes lb-agent boot,
# (3) external-IP allocator reusing an ENI EIP for the ALB. Re-enable once
# those land.
for workbook in bastion-private-subnet nginx-webserver s3-webapp; do
for workbook in nginx-alb bastion-private-subnet nginx-webserver s3-webapp; do
if ! run_workbook "$workbook"; then
log "FAIL ${workbook} — aborting remaining workbooks"
exit 1
fi
done
CURRENT_WORKBOOK=""
log "All workbooks passed"