mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
feat(e2e): bare-metal multi-node cluster support + density-aware guards (#227)
* feat(predastore): support 2-node distributed mode with RS(1,1)
Lower minimum node threshold from 3 to 2. Emit data=1,parity=1 when
exactly 2 nodes configured — effectively mirroring. 3+ nodes continue
to use RS(2,1).
* fix(ec2): route StartInstances back to original node with node-down fallback
ec2.start used a plain queue group, so NATS could route a restart request to
any daemon — including a capacity-constrained NUC that can't fit a second
instance. This caused InsufficientInstanceCapacity in the Phase 10 stop/start
test on 2-node bare-metal clusters where one node has significantly fewer
resources.
Each daemon now also subscribes to ec2.start.{nodeId} (no queue group). The
queue-group handler peeks at the instance ID, looks up LastNode from the
stopped KV, and forwards to ec2.start.{lastNode}. If the original node does
not respond within 5 seconds (down or partitioned), the handler falls back to
starting locally on whichever node picked up the request — preserving the
existing behaviour for node-failure scenarios.
The direct handler (ec2.start.{nodeId}) always starts locally and never
forwards, preventing routing loops.
* fix(ec2): fall back to local start when original node returns InsufficientInstanceCapacity
The forwarding handler was relaying all responses from the targeted node
unconditionally, including InsufficientInstanceCapacity. This meant that if
the NUC (node2) was the LastNode but had no remaining capacity, the error
propagated to the caller rather than falling back to the local node which may
have room.
Now only success and non-capacity errors are relayed; an InsufficientInstance-
Capacity response from the targeted node triggers a local start attempt, giving
any other node in the cluster a chance to absorb the instance.
* test(e2e): make VPC stop/start restart non-fatal on InsufficientInstanceCapacity
Asymmetric hardware (NUC node2) may not have capacity to restart all
VPC instances. IP-persists-while-stopped is still verified and fails
hard; restart verification is skipped with a warning on capacity errors.
* test(e2e): density-aware capacity guard across bare-metal multi-node suite
Sub-3-node bare-metal clusters with asymmetric/undersized nodes (e.g. a
4-vCPU NUC alongside a 16-vCPU box) can fit only one t3.nano on the
smallest node after the daemon's host reserve. Phases that direct or
indirectly land >1 instance on that node hit InsufficientInstanceCapacity
and previously hard-failed the entire suite.
Add is_density_skip helper and skip_test counter, and wrap every
density-sensitive run-instances / start-instances / wait-for-running call
site (phases 3, 5, 7, 10, 11) so capacity exhaustion on sub-3-node
clusters records a SKIP rather than a FAIL. On 3+ node clusters the
guard short-circuits to false, preserving existing failure semantics.
Summary block now prints Skipped count + names so degraded coverage is
visible in CI logs.
Refs: mulga-siv-72
* test(e2e): parameterise lb SSH key + user via PEER_SSH_KEY / PEER_SSH_USER
Allow run-lb-e2e.sh to run against clusters that don't use tf-user with the
tofu-cluster SSH key (e.g. bare-metal nodes which use spinifex@ + bm-ci key).
Defaults preserve existing behaviour on the tofu-cluster CI path.
Refs: mulga-siv-73
This commit is contained in:
Josh Siviour
@@ -993,8 +993,8 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
var predastoreNodeID int
|
||||
if predastoreNodesStr != "" {
|
||||
ips := strings.Split(predastoreNodesStr, ",")
|
||||
if len(ips) < 3 {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 3 IPs, got %d\n", len(ips))
|
||||
if len(ips) < 2 {
|
||||
fmt.Fprintf(os.Stderr, "❌ Error: --predastore-nodes requires at least 2 IPs, got %d\n", len(ips))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -1283,7 +1283,7 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
|
||||
|
||||
// Generate multi-node predastore config
|
||||
var predastoreNodeID int
|
||||
hasPredastoreConfig := len(predastoreNodes) >= 3
|
||||
hasPredastoreConfig := len(predastoreNodes) >= 2
|
||||
if hasPredastoreConfig {
|
||||
predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, accessKey, secretKey, region, natsToken, configDir, bindIP)
|
||||
if err != nil {
|
||||
@@ -1696,7 +1696,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
|
||||
|
||||
// Generate multi-node predastore config
|
||||
var predastoreNodeID int
|
||||
hasPredastoreConfig := len(predastoreNodes) >= 3
|
||||
hasPredastoreConfig := len(predastoreNodes) >= 2
|
||||
|
||||
if hasPredastoreConfig {
|
||||
predastoreContent, err := admin.GenerateMultiNodePredastoreConfig(predastoreMultiNodeTemplate, predastoreNodes, creds.AccessKey, creds.SecretKey, creds.Region, creds.NatsToken, configDir, bindIP)
|
||||
|
||||
@@ -9,8 +9,13 @@ port = 8443
|
||||
debug = false
|
||||
|
||||
[rs]
|
||||
{{- if eq (len .Nodes) 2}}
|
||||
data = 1
|
||||
parity = 1
|
||||
{{- else}}
|
||||
data = 2
|
||||
parity = 1
|
||||
{{- end}}
|
||||
{{- range .Nodes}}
|
||||
|
||||
[[db]]
|
||||
|
||||
@@ -886,8 +886,8 @@ func SetupAWSCredentials(accessKey, secretKey, region, certPath, bindIP, wanIP s
|
||||
// multi-node Predastore cluster. Each node gets its own DB entry (port 6660)
|
||||
// and shard entry (port 9991) on a distinct IP. Node ID 1 is the bootstrap leader.
|
||||
func GenerateMultiNodePredastoreConfig(templateStr string, nodes []PredastoreNodeConfig, accessKey, secretKey, region, natsToken, configDir, bindIP string) (string, error) {
|
||||
if len(nodes) < 3 {
|
||||
return "", fmt.Errorf("multi-node predastore requires at least 3 nodes, got %d", len(nodes))
|
||||
if len(nodes) < 2 {
|
||||
return "", fmt.Errorf("multi-node predastore requires at least 2 nodes, got %d", len(nodes))
|
||||
}
|
||||
|
||||
data := struct {
|
||||
|
||||
@@ -738,10 +738,9 @@ func TestGenerateMultiNodePredastoreConfig_MinimumNodes(t *testing.T) {
|
||||
|
||||
_, err := GenerateMultiNodePredastoreConfig(tmpl, []PredastoreNodeConfig{
|
||||
{ID: 1, Host: "10.0.0.1"},
|
||||
{ID: 2, Host: "10.0.0.2"},
|
||||
}, "AK", "SK", "us-east-1", "nats-token", "/config", "10.0.0.1")
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "at least 3 nodes")
|
||||
assert.Contains(t, err.Error(), "at least 2 nodes")
|
||||
}
|
||||
|
||||
func TestGenerateMultiNodePredastoreConfig_InvalidTemplate(t *testing.T) {
|
||||
|
||||
@@ -744,6 +744,7 @@ func (d *Daemon) subscribeAll() error {
|
||||
{"ec2.ModifyInstanceAttribute", d.handleEC2ModifyInstanceAttribute, "spinifex-workers"},
|
||||
{"ec2.DescribeInstanceAttribute", d.handleEC2DescribeInstanceAttribute, "spinifex-workers"},
|
||||
{"ec2.start", d.handleEC2StartStoppedInstance, "spinifex-workers"},
|
||||
{fmt.Sprintf("ec2.start.%s", d.node), d.handleEC2StartStoppedInstanceDirect, ""},
|
||||
{"ec2.terminate", d.handleEC2TerminateStoppedInstance, "spinifex-workers"},
|
||||
{"ec2.DescribeStoppedInstances", d.handleEC2DescribeStoppedInstances, "spinifex-workers"},
|
||||
{"ec2.DescribeTerminatedInstances", d.handleEC2DescribeTerminatedInstances, "spinifex-workers"},
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
@@ -15,6 +16,10 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// startStoppedForwardTimeout is how long the queue-group handler waits for the
|
||||
// node-targeted ec2.start.{nodeId} response before falling back to a local launch.
|
||||
const startStoppedForwardTimeout = 5 * time.Second
|
||||
|
||||
// handleEC2RunInstances orchestrates the RunInstances flow across
|
||||
// InstanceService.PrepareRunInstances (validation + ENI creation),
|
||||
// daemon-local Insert/WriteState/per-instance subscribe, and
|
||||
@@ -318,8 +323,66 @@ func (d *Daemon) handleEC2DescribeInstanceStatus(msg *nats.Msg) {
|
||||
handleNATSRequest(msg, d.instanceService.DescribeInstanceStatus)
|
||||
}
|
||||
|
||||
// startStoppedInstanceRequest is the payload for ec2.start topic
|
||||
// handleEC2StartStoppedInstance handles the generic ec2.start queue-group topic.
|
||||
// It reads the stopped instance's LastNode from shared KV and forwards the
|
||||
// request to ec2.start.{lastNode} when the instance last ran on a different
|
||||
// node. This keeps instances on their original node so the per-node resource
|
||||
// manager sees the correct allocation. If the target node does not respond
|
||||
// within startStoppedForwardTimeout (node down, not yet recovered), the handler
|
||||
// falls back to starting the instance locally on whatever node picked this up.
|
||||
func (d *Daemon) handleEC2StartStoppedInstance(msg *nats.Msg) {
|
||||
// Peek at the instance ID without full unmarshal — we only need it for the
|
||||
// LastNode lookup. The full unmarshal happens inside StartStoppedInstance.
|
||||
var peek struct {
|
||||
InstanceID string `json:"instance_id"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &peek); err != nil || peek.InstanceID == "" {
|
||||
// Can't determine target node — fall through to local start which will
|
||||
// return the appropriate error (missing parameter / unmarshal failure).
|
||||
handleNATSRequest(msg, d.instanceService.StartStoppedInstance)
|
||||
return
|
||||
}
|
||||
|
||||
lastNode := d.instanceService.StoppedInstanceNode(peek.InstanceID)
|
||||
if lastNode != "" && lastNode != d.node {
|
||||
targetTopic := fmt.Sprintf("ec2.start.%s", lastNode)
|
||||
forwardMsg := nats.NewMsg(targetTopic)
|
||||
forwardMsg.Data = msg.Data
|
||||
forwardMsg.Header.Set(utils.AccountIDHeader, utils.AccountIDFromMsg(msg))
|
||||
|
||||
resp, err := d.natsConn.RequestMsg(forwardMsg, startStoppedForwardTimeout)
|
||||
if err == nil {
|
||||
// ValidateErrorPayload returns a non-nil error when the payload IS an
|
||||
// AWS error response; nil means it is a success payload.
|
||||
errPayload, isErrPayload := utils.ValidateErrorPayload(resp.Data)
|
||||
isCapacity := isErrPayload != nil &&
|
||||
errPayload.Code != nil &&
|
||||
*errPayload.Code == awserrors.ErrorInsufficientInstanceCapacity
|
||||
|
||||
if !isCapacity {
|
||||
// Relay success or any non-capacity error as-is.
|
||||
if relayErr := msg.Respond(resp.Data); relayErr != nil {
|
||||
slog.Error("ec2.start: failed to relay response from original node",
|
||||
"instanceId", peek.InstanceID, "lastNode", lastNode, "err", relayErr)
|
||||
}
|
||||
return
|
||||
}
|
||||
slog.Warn("ec2.start: original node at capacity, starting locally",
|
||||
"instanceId", peek.InstanceID, "lastNode", lastNode)
|
||||
} else {
|
||||
slog.Warn("ec2.start: original node unreachable, starting locally",
|
||||
"instanceId", peek.InstanceID, "lastNode", lastNode, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
handleNATSRequest(msg, d.instanceService.StartStoppedInstance)
|
||||
}
|
||||
|
||||
// handleEC2StartStoppedInstanceDirect handles ec2.start.{nodeId} — the
|
||||
// node-targeted topic used by handleEC2StartStoppedInstance to forward start
|
||||
// requests back to the original node. Always starts locally; never forwards
|
||||
// further, preventing routing loops.
|
||||
func (d *Daemon) handleEC2StartStoppedInstanceDirect(msg *nats.Msg) {
|
||||
handleNATSRequest(msg, d.instanceService.StartStoppedInstance)
|
||||
}
|
||||
|
||||
|
||||
@@ -1790,6 +1790,21 @@ func (s *InstanceServiceImpl) ModifyInstanceAttribute(input *ec2.ModifyInstanceA
|
||||
return &ec2.ModifyInstanceAttributeOutput{}, nil
|
||||
}
|
||||
|
||||
// StoppedInstanceNode returns the LastNode field stored in the shared KV for a
|
||||
// stopped instance. Returns "" when the store is unavailable, the instance is
|
||||
// not found, or the entry has no LastNode set. Used by the daemon's ec2.start
|
||||
// handler to route start requests back to the original node when possible.
|
||||
func (s *InstanceServiceImpl) StoppedInstanceNode(instanceID string) string {
|
||||
if s.stoppedStore == nil {
|
||||
return ""
|
||||
}
|
||||
instance, err := s.stoppedStore.LoadStoppedInstance(instanceID)
|
||||
if err != nil || instance == nil {
|
||||
return ""
|
||||
}
|
||||
return instance.LastNode
|
||||
}
|
||||
|
||||
// StartStoppedInstance picks up a stopped instance from shared KV, re-launches
|
||||
// it on this daemon node, then removes it from shared KV.
|
||||
func (s *InstanceServiceImpl) StartStoppedInstance(input *StartStoppedInstanceInput, accountID string) (*StartStoppedInstanceOutput, error) {
|
||||
|
||||
@@ -45,7 +45,17 @@ export AWS_CA_BUNDLE="/etc/spinifex/ca.pem"
|
||||
# Track test results
|
||||
TESTS_PASSED=0
|
||||
TESTS_FAILED=0
|
||||
TESTS_SKIPPED=0
|
||||
FAILED_TESTS=()
|
||||
SKIPPED_TESTS=()
|
||||
|
||||
# Cluster size target: phases that depend on multi-instance density (3+
|
||||
# t3.nano instances spread across the cluster) require NODE_COUNT >= DENSITY_NODE_TARGET.
|
||||
# On smaller clusters the smallest node (e.g. a 4-vCPU NUC) can hold only one
|
||||
# t3.nano after the daemon's host reserve, so any phase that puts >1 instance
|
||||
# on it — directly via spread or indirectly via StartStoppedInstance returning
|
||||
# instances to their original node — legitimately exhausts capacity.
|
||||
DENSITY_NODE_TARGET=3
|
||||
|
||||
# ==========================================================================
|
||||
# Helper functions
|
||||
@@ -376,6 +386,22 @@ fail_test() {
|
||||
FAILED_TESTS+=("$name")
|
||||
}
|
||||
|
||||
skip_test() {
|
||||
local name="$1"
|
||||
local reason="${2:-density skip}"
|
||||
echo " $name SKIPPED ($reason)"
|
||||
TESTS_SKIPPED=$((TESTS_SKIPPED + 1))
|
||||
SKIPPED_TESTS+=("$name ($reason)")
|
||||
}
|
||||
|
||||
# is_density_skip returns 0 (true) when $1 contains InsufficientInstanceCapacity
|
||||
# AND the cluster is smaller than the design target (3 nodes). On larger clusters
|
||||
# capacity exhaustion is a real failure, not a hardware-sizing artefact.
|
||||
is_density_skip() {
|
||||
[ "$NODE_COUNT" -lt "$DENSITY_NODE_TARGET" ] || return 1
|
||||
echo "$1" | grep -q "InsufficientInstanceCapacity"
|
||||
}
|
||||
|
||||
# ==========================================================================
|
||||
# EXIT trap — dump logs on failure
|
||||
# ==========================================================================
|
||||
@@ -558,12 +584,27 @@ echo " Default SG ingress: tcp/22 from 0.0.0.0/0"
|
||||
|
||||
# Launch 3 instances with stagger to encourage distribution
|
||||
echo "Launching 3 instances..."
|
||||
PHASE3_DENSITY_SKIP=false
|
||||
for i in 1 2 3; do
|
||||
echo " Launching instance $i..."
|
||||
set +e
|
||||
RUN_OUTPUT=$($AWS_EC2 run-instances \
|
||||
--image-id "$AMI_ID" \
|
||||
--instance-type "$INSTANCE_TYPE" \
|
||||
--key-name spinifex-key)
|
||||
--key-name spinifex-key 2>&1)
|
||||
RUN_RC=$?
|
||||
set -e
|
||||
|
||||
if [ $RUN_RC -ne 0 ]; then
|
||||
if is_density_skip "$RUN_OUTPUT"; then
|
||||
skip_test "Phase 3 launch instance $i" "InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT"
|
||||
PHASE3_DENSITY_SKIP=true
|
||||
break
|
||||
fi
|
||||
echo " ERROR: Failed to launch instance $i"
|
||||
echo " Output: $RUN_OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
INSTANCE_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId')
|
||||
if [ -z "$INSTANCE_ID" ] || [ "$INSTANCE_ID" == "null" ]; then
|
||||
@@ -578,15 +619,29 @@ for i in 1 2 3; do
|
||||
[ $i -lt 3 ] && sleep 2
|
||||
done
|
||||
|
||||
# Wait for all instances to be running
|
||||
if [ "$PHASE3_DENSITY_SKIP" = true ] && [ ${#INSTANCE_IDS[@]} -eq 0 ]; then
|
||||
echo " Phase 3 launched zero instances on undersized cluster; skipping dependent phases gracefully."
|
||||
fi
|
||||
|
||||
# Wait for all instances to be running. On a sub-DENSITY_NODE_TARGET cluster
|
||||
# an instance that failed to start (capacity, scheduler race) is downgraded
|
||||
# to a skip rather than a hard exit so the rest of the suite still runs.
|
||||
echo ""
|
||||
echo "Waiting for instances to reach running state..."
|
||||
RUNNING_INSTANCE_IDS=()
|
||||
for instance_id in "${INSTANCE_IDS[@]}"; do
|
||||
wait_for_instance_state "$instance_id" "running" 60 || {
|
||||
echo "ERROR: Instance $instance_id failed to start"
|
||||
exit 1
|
||||
}
|
||||
if wait_for_instance_state "$instance_id" "running" 60; then
|
||||
RUNNING_INSTANCE_IDS+=("$instance_id")
|
||||
continue
|
||||
fi
|
||||
if [ "$NODE_COUNT" -lt "$DENSITY_NODE_TARGET" ]; then
|
||||
skip_test "Phase 3 wait-running $instance_id" "instance never reached running on NODE_COUNT=$NODE_COUNT"
|
||||
continue
|
||||
fi
|
||||
echo "ERROR: Instance $instance_id failed to start"
|
||||
exit 1
|
||||
done
|
||||
INSTANCE_IDS=("${RUNNING_INSTANCE_IDS[@]}")
|
||||
|
||||
# Check distribution via spx get vms or QEMU process check
|
||||
echo ""
|
||||
@@ -741,6 +796,10 @@ echo "========================================"
|
||||
SPINIFEX_AZ=$($AWS_EC2 describe-availability-zones --query 'AvailabilityZones[0].ZoneName' --output text)
|
||||
echo "AZ: $SPINIFEX_AZ"
|
||||
|
||||
if [ ${#INSTANCE_IDS[@]} -eq 0 ]; then
|
||||
skip_test "Volume lifecycle" "no Phase 3 instances available for attach"
|
||||
else
|
||||
|
||||
# Create volume
|
||||
echo "Creating 10GB test volume..."
|
||||
CREATE_OUTPUT=$($AWS_EC2 create-volume --size 10 --availability-zone "$SPINIFEX_AZ")
|
||||
@@ -826,6 +885,8 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
fi # INSTANCE_IDS non-empty guard
|
||||
|
||||
echo ""
|
||||
|
||||
# ==========================================================================
|
||||
@@ -861,40 +922,55 @@ echo "Phase 7: Cross-Node Operations"
|
||||
echo "========================================"
|
||||
echo "Testing stop/start via a gateway on a DIFFERENT node than the hosting node..."
|
||||
|
||||
# Pick first instance and find its host
|
||||
TEST_INSTANCE="${INSTANCE_IDS[0]}"
|
||||
INSTANCE_HOST=$(find_instance_node "$TEST_INSTANCE" || echo "$LOCAL_IP")
|
||||
echo " Instance $TEST_INSTANCE is on $INSTANCE_HOST"
|
||||
if [ ${#INSTANCE_IDS[@]} -eq 0 ]; then
|
||||
skip_test "Cross-node stop/start" "no Phase 3 instances available"
|
||||
else
|
||||
# Pick first instance and find its host
|
||||
TEST_INSTANCE="${INSTANCE_IDS[0]}"
|
||||
INSTANCE_HOST=$(find_instance_node "$TEST_INSTANCE" || echo "$LOCAL_IP")
|
||||
echo " Instance $TEST_INSTANCE is on $INSTANCE_HOST"
|
||||
|
||||
# Pick a different node's gateway for the operation
|
||||
OTHER_GW=""
|
||||
for ip in "${NODE_IPS[@]}"; do
|
||||
if [ "$ip" != "$INSTANCE_HOST" ]; then
|
||||
OTHER_GW="$ip"
|
||||
break
|
||||
# Pick a different node's gateway for the operation
|
||||
OTHER_GW=""
|
||||
for ip in "${NODE_IPS[@]}"; do
|
||||
if [ "$ip" != "$INSTANCE_HOST" ]; then
|
||||
OTHER_GW="$ip"
|
||||
break
|
||||
fi
|
||||
done
|
||||
echo " Will operate via gateway on $OTHER_GW"
|
||||
|
||||
# Stop via other gateway
|
||||
echo " Stopping instance via $OTHER_GW..."
|
||||
aws_via "$OTHER_GW" ec2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null
|
||||
wait_for_instance_state "$TEST_INSTANCE" "stopped" 60 "$OTHER_GW"
|
||||
|
||||
# Pick yet another gateway for start (or same other if only 2 choices)
|
||||
THIRD_GW=""
|
||||
for ip in "${NODE_IPS[@]}"; do
|
||||
if [ "$ip" != "$INSTANCE_HOST" ] && [ "$ip" != "$OTHER_GW" ]; then
|
||||
THIRD_GW="$ip"
|
||||
break
|
||||
fi
|
||||
done
|
||||
THIRD_GW="${THIRD_GW:-$OTHER_GW}"
|
||||
echo " Starting instance via $THIRD_GW..."
|
||||
set +e
|
||||
START_OUT=$(aws_via "$THIRD_GW" ec2 start-instances --instance-ids "$TEST_INSTANCE" 2>&1)
|
||||
START_RC=$?
|
||||
set -e
|
||||
if [ $START_RC -ne 0 ]; then
|
||||
if is_density_skip "$START_OUT"; then
|
||||
skip_test "Cross-node stop/start" "InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT"
|
||||
else
|
||||
echo " ERROR: start-instances failed: $START_OUT"
|
||||
fail_test "Cross-node stop/start"
|
||||
fi
|
||||
else
|
||||
wait_for_instance_state "$TEST_INSTANCE" "running" 60 "$THIRD_GW"
|
||||
pass_test "Cross-node stop/start"
|
||||
fi
|
||||
done
|
||||
echo " Will operate via gateway on $OTHER_GW"
|
||||
|
||||
# Stop via other gateway
|
||||
echo " Stopping instance via $OTHER_GW..."
|
||||
aws_via "$OTHER_GW" ec2 stop-instances --instance-ids "$TEST_INSTANCE" > /dev/null
|
||||
wait_for_instance_state "$TEST_INSTANCE" "stopped" 60 "$OTHER_GW"
|
||||
|
||||
# Pick yet another gateway for start (or same other if only 2 choices)
|
||||
THIRD_GW=""
|
||||
for ip in "${NODE_IPS[@]}"; do
|
||||
if [ "$ip" != "$INSTANCE_HOST" ] && [ "$ip" != "$OTHER_GW" ]; then
|
||||
THIRD_GW="$ip"
|
||||
break
|
||||
fi
|
||||
done
|
||||
THIRD_GW="${THIRD_GW:-$OTHER_GW}"
|
||||
echo " Starting instance via $THIRD_GW..."
|
||||
aws_via "$THIRD_GW" ec2 start-instances --instance-ids "$TEST_INSTANCE" > /dev/null
|
||||
wait_for_instance_state "$TEST_INSTANCE" "running" 60 "$THIRD_GW"
|
||||
|
||||
pass_test "Cross-node stop/start"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
@@ -1082,13 +1158,29 @@ else
|
||||
echo "Launching 3 VPC instances..."
|
||||
VPC_INSTANCE_IDS=()
|
||||
VPC_LAUNCH_OK=true
|
||||
VPC_DENSITY_SKIP=false
|
||||
for i in 1 2 3; do
|
||||
echo " Launching VPC instance $i with subnet $SUBNET_ID..."
|
||||
set +e
|
||||
RUN_OUTPUT=$($AWS_EC2 run-instances \
|
||||
--image-id "$AMI_ID" \
|
||||
--instance-type "$INSTANCE_TYPE" \
|
||||
--key-name spinifex-key \
|
||||
--subnet-id "$SUBNET_ID")
|
||||
--subnet-id "$SUBNET_ID" 2>&1)
|
||||
VPC_RUN_RC=$?
|
||||
set -e
|
||||
|
||||
if [ $VPC_RUN_RC -ne 0 ]; then
|
||||
if is_density_skip "$RUN_OUTPUT"; then
|
||||
skip_test "Phase 10 VPC instance $i launch" "InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT"
|
||||
VPC_DENSITY_SKIP=true
|
||||
VPC_LAUNCH_OK=false
|
||||
break
|
||||
fi
|
||||
echo " ERROR: Failed to launch VPC instance $i: $RUN_OUTPUT"
|
||||
VPC_LAUNCH_OK=false
|
||||
break
|
||||
fi
|
||||
|
||||
VPC_INST_ID=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].InstanceId')
|
||||
VPC_INST_IP=$(echo "$RUN_OUTPUT" | jq -r '.Instances[0].PrivateIpAddress // empty')
|
||||
@@ -1202,29 +1294,52 @@ else
|
||||
fi
|
||||
done
|
||||
|
||||
# Restart all VPC instances
|
||||
# Restart all VPC instances. InsufficientInstanceCapacity here is
|
||||
# downgraded to a skip only on sub-DENSITY_NODE_TARGET clusters;
|
||||
# on a 3+ node cluster it remains a real failure.
|
||||
VPC_RESTART_OK=true
|
||||
VPC_RESTART_DENSITY_SKIP=false
|
||||
set +e
|
||||
for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
|
||||
$AWS_EC2 start-instances --instance-ids "$vpc_inst" > /dev/null
|
||||
done
|
||||
for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
|
||||
wait_for_instance_state "$vpc_inst" "running" 60 || true
|
||||
done
|
||||
|
||||
# Verify IPs persist after restart
|
||||
for idx in "${!VPC_INSTANCE_IDS[@]}"; do
|
||||
vpc_inst="${VPC_INSTANCE_IDS[$idx]}"
|
||||
expected_ip="${VPC_PRIVATE_IPS[$idx]}"
|
||||
RESTARTED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \
|
||||
--query 'Reservations[0].Instances[0].PrivateIpAddress' --output text)
|
||||
if [ "$RESTARTED_IP" != "$expected_ip" ]; then
|
||||
echo " ERROR: $vpc_inst IP changed after restart (expected $expected_ip, got $RESTARTED_IP)"
|
||||
IP_PERSIST_OK=false
|
||||
else
|
||||
echo " $vpc_inst: IP=$RESTARTED_IP (matches pre-stop)"
|
||||
START_OUT=$($AWS_EC2 start-instances --instance-ids "$vpc_inst" 2>&1)
|
||||
START_RC=$?
|
||||
if [ $START_RC -ne 0 ]; then
|
||||
if is_density_skip "$START_OUT"; then
|
||||
echo " SKIP: $vpc_inst could not restart (InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT)"
|
||||
VPC_RESTART_OK=false
|
||||
VPC_RESTART_DENSITY_SKIP=true
|
||||
else
|
||||
echo " ERROR: $vpc_inst start-instances failed: $START_OUT"
|
||||
IP_PERSIST_OK=false
|
||||
VPC_RESTART_OK=false
|
||||
fi
|
||||
fi
|
||||
done
|
||||
set -e
|
||||
|
||||
if [ "$IP_PERSIST_OK" = true ]; then
|
||||
if [ "$VPC_RESTART_OK" = true ]; then
|
||||
for vpc_inst in "${VPC_INSTANCE_IDS[@]}"; do
|
||||
wait_for_instance_state "$vpc_inst" "running" 60 || true
|
||||
done
|
||||
|
||||
# Verify IPs persist after restart
|
||||
for idx in "${!VPC_INSTANCE_IDS[@]}"; do
|
||||
vpc_inst="${VPC_INSTANCE_IDS[$idx]}"
|
||||
expected_ip="${VPC_PRIVATE_IPS[$idx]}"
|
||||
RESTARTED_IP=$($AWS_EC2 describe-instances --instance-ids "$vpc_inst" \
|
||||
--query 'Reservations[0].Instances[0].PrivateIpAddress' --output text)
|
||||
if [ "$RESTARTED_IP" != "$expected_ip" ]; then
|
||||
echo " ERROR: $vpc_inst IP changed after restart (expected $expected_ip, got $RESTARTED_IP)"
|
||||
IP_PERSIST_OK=false
|
||||
else
|
||||
echo " $vpc_inst: IP=$RESTARTED_IP (matches pre-stop)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$VPC_RESTART_DENSITY_SKIP" = true ]; then
|
||||
skip_test "VPC stop/start IP persistence" "restart blocked by InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT"
|
||||
elif [ "$IP_PERSIST_OK" = true ]; then
|
||||
pass_test "VPC stop/start IP persistence"
|
||||
else
|
||||
fail_test "VPC stop/start IP persistence"
|
||||
@@ -1240,7 +1355,11 @@ else
|
||||
echo "Cleaning up VPC instances..."
|
||||
terminate_and_wait "${VPC_INSTANCE_IDS[@]}" || true
|
||||
else
|
||||
fail_test "VPC instance launch"
|
||||
if [ "$VPC_DENSITY_SKIP" = true ]; then
|
||||
skip_test "VPC instance launch" "InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT"
|
||||
else
|
||||
fail_test "VPC instance launch"
|
||||
fi
|
||||
# Cleanup any partially-launched VPC instances
|
||||
if [ ${#VPC_INSTANCE_IDS[@]} -gt 0 ]; then
|
||||
echo " Cleaning up partially-launched VPC instances..."
|
||||
@@ -1321,12 +1440,43 @@ echo " Private SG: $NATGW_PRIV_SG (tcp/22 from bastion-sg, icmp from VPC)"
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Launching bastion in public subnet..."
|
||||
NATGW_BASTION_ID=$($AWS_EC2 run-instances \
|
||||
set +e
|
||||
BASTION_RUN_OUT=$($AWS_EC2 run-instances \
|
||||
--image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" \
|
||||
--subnet-id "$NATGW_PUB_SUBNET" --key-name spinifex-key \
|
||||
--security-group-ids "$NATGW_BASTION_SG" \
|
||||
--count 1 --query 'Instances[0].InstanceId' --output text)
|
||||
--count 1 --query 'Instances[0].InstanceId' --output text 2>&1)
|
||||
BASTION_RUN_RC=$?
|
||||
set -e
|
||||
NATGW_DENSITY_SKIP=false
|
||||
if [ $BASTION_RUN_RC -ne 0 ]; then
|
||||
if is_density_skip "$BASTION_RUN_OUT"; then
|
||||
skip_test "NAT GW bastion launch" "InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT"
|
||||
NATGW_DENSITY_SKIP=true
|
||||
NATGW_BASTION_ID=""
|
||||
else
|
||||
echo " ERROR: bastion run-instances failed: $BASTION_RUN_OUT"
|
||||
fail_test "NAT GW bastion launch"
|
||||
NATGW_BASTION_ID=""
|
||||
fi
|
||||
else
|
||||
NATGW_BASTION_ID="$BASTION_RUN_OUT"
|
||||
fi
|
||||
echo " Bastion: $NATGW_BASTION_ID"
|
||||
if [ -z "$NATGW_BASTION_ID" ]; then
|
||||
# Density-skip or hard fail at bastion launch: roll back the Phase 11 VPC
|
||||
# plumbing and skip the rest of the phase.
|
||||
$AWS_EC2 delete-security-group --group-id "$NATGW_PRIV_SG" 2>/dev/null || true
|
||||
$AWS_EC2 delete-security-group --group-id "$NATGW_BASTION_SG" 2>/dev/null || true
|
||||
$AWS_EC2 delete-subnet --subnet-id "$NATGW_PRIV_SUBNET" 2>/dev/null || true
|
||||
$AWS_EC2 delete-subnet --subnet-id "$NATGW_PUB_SUBNET" 2>/dev/null || true
|
||||
$AWS_EC2 detach-internet-gateway --vpc-id "$NATGW_VPC_ID" \
|
||||
--internet-gateway-id "$NATGW_IGW_ID" 2>/dev/null || true
|
||||
$AWS_EC2 delete-internet-gateway --internet-gateway-id "$NATGW_IGW_ID" 2>/dev/null || true
|
||||
$AWS_EC2 delete-vpc --vpc-id "$NATGW_VPC_ID" 2>/dev/null || true
|
||||
echo " Phase 11 short-circuited after bastion launch failure"
|
||||
else
|
||||
|
||||
wait_for_instance_state "$NATGW_BASTION_ID" "running" 120
|
||||
|
||||
NATGW_BASTION_PUB_IP=$($AWS_EC2 describe-instances \
|
||||
@@ -1386,24 +1536,44 @@ else
|
||||
echo " Placement group: nat-spread (spread)"
|
||||
|
||||
# Launch NODE_COUNT instances in private subnet with spread placement
|
||||
set +e
|
||||
NATGW_PRIV_OUTPUT=$($AWS_EC2 run-instances \
|
||||
--image-id "$AMI_ID" --instance-type "$INSTANCE_TYPE" \
|
||||
--subnet-id "$NATGW_PRIV_SUBNET" --key-name spinifex-key \
|
||||
--security-group-ids "$NATGW_PRIV_SG" \
|
||||
--count "$NODE_COUNT" --placement "GroupName=nat-spread" --output json)
|
||||
--count "$NODE_COUNT" --placement "GroupName=nat-spread" --output json 2>&1)
|
||||
NATGW_PRIV_RC=$?
|
||||
set -e
|
||||
NATGW_PRIV_IDS=()
|
||||
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
||||
NATGW_PRIV_IDS+=("$(echo "$NATGW_PRIV_OUTPUT" | jq -r ".Instances[$i].InstanceId")")
|
||||
done
|
||||
echo " Private instances: ${NATGW_PRIV_IDS[*]}"
|
||||
NATGW_PRIV_DENSITY_SKIP=false
|
||||
if [ $NATGW_PRIV_RC -ne 0 ]; then
|
||||
if is_density_skip "$NATGW_PRIV_OUTPUT"; then
|
||||
skip_test "Spread placement private launch" "InsufficientInstanceCapacity on NODE_COUNT=$NODE_COUNT"
|
||||
NATGW_PRIV_DENSITY_SKIP=true
|
||||
else
|
||||
echo " ERROR: spread run-instances failed: $NATGW_PRIV_OUTPUT"
|
||||
fail_test "Spread placement private launch"
|
||||
fi
|
||||
else
|
||||
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
||||
NATGW_PRIV_IDS+=("$(echo "$NATGW_PRIV_OUTPUT" | jq -r ".Instances[$i].InstanceId")")
|
||||
done
|
||||
fi
|
||||
echo " Private instances: ${NATGW_PRIV_IDS[*]:-(none)}"
|
||||
|
||||
# Wait for all running
|
||||
ALL_RUNNING=true
|
||||
for inst_id in "${NATGW_PRIV_IDS[@]}"; do
|
||||
if ! wait_for_instance_state "$inst_id" "running" 120; then
|
||||
ALL_RUNNING=false
|
||||
fi
|
||||
done
|
||||
if [ ${#NATGW_PRIV_IDS[@]} -eq 0 ]; then
|
||||
# Nothing was launched (density skip or other failure); short-circuit
|
||||
# the spread/NAT GW assertions but keep cleanup.
|
||||
ALL_RUNNING=false
|
||||
else
|
||||
for inst_id in "${NATGW_PRIV_IDS[@]}"; do
|
||||
if ! wait_for_instance_state "$inst_id" "running" 120; then
|
||||
ALL_RUNNING=false
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$ALL_RUNNING" = true ]; then
|
||||
echo ""
|
||||
@@ -1574,7 +1744,11 @@ else
|
||||
fail_test "Private instance SSH via bastion"
|
||||
fi
|
||||
else
|
||||
fail_test "Private instance launch (spread)"
|
||||
if [ "$NATGW_PRIV_DENSITY_SKIP" = true ]; then
|
||||
: # already recorded via skip_test "Spread placement private launch"
|
||||
else
|
||||
fail_test "Private instance launch (spread)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Cleanup Phase 11 instances
|
||||
@@ -1601,6 +1775,8 @@ else
|
||||
echo " Phase 11 cleanup complete"
|
||||
fi
|
||||
|
||||
fi # NATGW_BASTION_ID non-empty guard
|
||||
|
||||
echo ""
|
||||
|
||||
# ==========================================================================
|
||||
@@ -1623,14 +1799,21 @@ echo ""
|
||||
echo "========================================"
|
||||
echo "Real Multi-Node E2E Test Summary"
|
||||
echo "========================================"
|
||||
echo " Passed: $TESTS_PASSED"
|
||||
echo " Failed: $TESTS_FAILED"
|
||||
echo " Passed: $TESTS_PASSED"
|
||||
echo " Failed: $TESTS_FAILED"
|
||||
echo " Skipped: $TESTS_SKIPPED"
|
||||
if [ ${#FAILED_TESTS[@]} -gt 0 ]; then
|
||||
echo " Failed tests:"
|
||||
for t in "${FAILED_TESTS[@]}"; do
|
||||
echo " - $t"
|
||||
done
|
||||
fi
|
||||
if [ ${#SKIPPED_TESTS[@]} -gt 0 ]; then
|
||||
echo " Skipped tests:"
|
||||
for t in "${SKIPPED_TESTS[@]}"; do
|
||||
echo " - $t"
|
||||
done
|
||||
fi
|
||||
echo "========================================"
|
||||
|
||||
if [ $TESTS_FAILED -gt 0 ]; then
|
||||
@@ -1638,5 +1821,9 @@ if [ $TESTS_FAILED -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All tests passed!"
|
||||
if [ $TESTS_SKIPPED -gt 0 ]; then
|
||||
echo "All tests passed (with $TESTS_SKIPPED skipped — undersized cluster)"
|
||||
else
|
||||
echo "All tests passed!"
|
||||
fi
|
||||
exit 0
|
||||
|
||||
@@ -48,7 +48,8 @@ export AWS_PROFILE=spinifex
|
||||
AWS_EC2="aws --endpoint-url ${ENDPOINT} ec2"
|
||||
AWS_ELBV2="aws --endpoint-url ${ENDPOINT} elbv2"
|
||||
|
||||
SSH_KEY_PATH="$HOME/.ssh/tf-user-ap-southeast-2"
|
||||
SSH_KEY_PATH="${PEER_SSH_KEY:-$HOME/.ssh/tf-user-ap-southeast-2}"
|
||||
PEER_USER="${PEER_SSH_USER:-tf-user}"
|
||||
|
||||
PASSED=0
|
||||
FAILED=0
|
||||
@@ -70,7 +71,7 @@ peer_ssh() {
|
||||
-o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=10 \
|
||||
-o LogLevel=ERROR \
|
||||
"tf-user@${ip}" "$@"
|
||||
"${PEER_USER}@${ip}" "$@"
|
||||
}
|
||||
|
||||
# find_lb_host <eni_id> — discover which node hosts the LB system instance by
|
||||
|
||||
Reference in New Issue
Block a user